pytest/lib/palo

#!/bin/env python # -*- coding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ########################################################################### # # @file palo_client.py # @date 2015/02/04 15:26:21 # @brief Palo client # ############################################################################ """ Palo client for Palo2 testing. """ # 路径设置 import sys import os # 系统库 import time import logging import socket import pycurl from io import BytesIO import pytest import json import pymysql # local库 import util import palo_logger import palo_job from palo_config import BrokerInfo from palo_exception import PaloException from palo_exception import PaloClientException from palo_sql import * from palo_verify import verify, verify_by_sql # 日志异常对象 LOG = palo_logger.Logger.getLogger() L = palo_logger.StructedLogMessage def get_client(host_name, port, database_name=None, user='root', password='', charset='utf8', retry=True, http_port=None): """get client """ client = PaloClient(host_name, port, database_name, user, password, charset, http_port) if client.init(retry): return client else: raise PaloClientException('get client error: host:%s port:%s database:%s ' 'user:%s password:%s' % (host_name, port, database_name, user, password)) class PaloClient(object): """Palo client. 成员变量有: host port user password database_name charset connection """ def __init__(self, host, port, database_name=None, user='root', password='', charset='utf8', http_port=None): """ Connect to Palo FE. """ # TODO 成员变量私有化 self.host = host self.port = port self.user = user self.password = password self.database_name = database_name self.charset = charset self.connection = None self.cluster_name = 'default_cluster' if http_port is None: self.http_port = port - 1000 else: self.http_port = http_port self.palo_job_map = {} def init(self, retry=True): """init """ assert self.host retry_times = 0 while retry_times < 10: try: self.connection = pymysql.connect( host=self.host, port=self.port, user=self.user, passwd=self.password, charset=self.charset) break except Exception as e: LOG.error(L('Connect to Palo error', fe=str(self), error=e)) if not retry: return False retry_times += 1 time.sleep(1) else: LOG.error(L('Connect to Palo error', retry_times=retry_times)) return False LOG.info(L("Connected to Palo", host=self.host, port=self.port, user=self.user)) database_name = self.database_name if database_name is None: return True database_name_list = self.get_database_list() if database_name not in database_name_list: if not self.create_database(database_name): return False self.use(database_name) return True def __str__(self): return "[host: %s, port: %d, database: %s]" % (self.host, self.port, self.database_name) def __del__(self): if self.connection: self.connection.close() def __execute_and_rebuild_meta_class(self, sql, palo_job_cls): if palo_job_cls.__name__ in self.palo_job_map.keys(): result = self.execute(sql) else: cursor, result = self.execute(sql, return_cursor=True) description = cursor.description idx = 0 for col_info in description: setattr(palo_job_cls, col_info[0], idx) idx += 1 self.palo_job_map[palo_job_cls.__name__] = True return result def use(self, database_name): """ specify default database. """ try: self.database_name = database_name self.connection.select_db(database_name) except Exception as error: LOG.error(L("USE database error", fe=str(self), database_name=database_name, \ error=error)) return False return True def connect(self): """ 建立连接 """ try: self.connection = pymysql.connect( host=self.host, port=self.port, user=self.user, passwd=self.password, charset=self.charset) except: LOG.info(L("Re-connected to Palo FE fail.", host=self.host, port=self.port, user=self.user)) return False LOG.info(L("Re-connected to Palo FE success.", host=self.host, port=self.port, user=self.user)) return True def execute(self, sql, return_rows=False, return_cursor=False): """ 执行sql语句 """ if 'PALO_CLIENT_LOG_SQL' in os.environ.keys(): LOG.info(L("execute SQL", sql=sql)) if 'PALO_CLIENT_STDOUT' in os.environ.keys(): # if sql.upper().startswith('SHOW') or sql.upper().startswith('SELECT'): if sql.upper().startswith('SHOW PROC'): pass else: print(sql.strip(';') + ';') # avoid: table state is not normal, do not allow doing ALTER ops. if sql.upper().startswith('ALTER'): LOG.info(L("before alter sleep 0.5s")) time.sleep(0.5) cursor = self.connection.cursor() try: rows = cursor.execute(sql) except pymysql.err.InternalError: self.connection.ping(reconnect=True) rows = cursor.execute(sql) if not return_rows and not return_cursor: return cursor.fetchall() elif return_rows: return rows, cursor.fetchall() elif return_cursor: return cursor, cursor.fetchall() else: return cursor.fetchall() def verify(self, expected_file_list, table_name, database_name=None, save_file_list=None, encoding='utf8', cluster_name=None, key_desc='aggregate'): """ verify """ LOG.info(L("check file:", file=expected_file_list)) database_name = self.database_name if database_name is None else database_name # 获取的表的schema，desc schema = self.desc_table(table_name, database_name) data = self.select_all(table_name, database_name) return verify(expected_file_list, data, schema, table_name, database_name, encoding, save_file_list) def verify_by_sql(self, expected_file_list, sql, schema, save_file_list=None): """verify by sql""" LOG.info(L("check file:", file=expected_file_list)) # 获取的表的schema，desc data = self.execute(sql) return verify_by_sql(expected_file_list, data, schema, 'check_by_sql', self.database_name, None, save_file_list) def create_database(self, database_name, cluster_name=None): """ 创建database """ sql = "CREATE DATABASE %s" % database_name self.execute(sql) if not cluster_name: cluster_name = self.cluster_name database_name_list = self.get_database_list(cluster_name) self.use(database_name) if database_name in database_name_list: self.database_name = database_name LOG.info(L("CREATE DATABASE succ", database_name=database_name, fe=str(self))) return True else: LOG.warning(L("CREATE DATABASE fail", database_name=database_name, fe=str(self))) return False def create_table(self, table_name, column_list, partition_info=None, distribution_info=None, storage_type=None, storage_medium=None, storage_cooldown_time=None, bloom_filter_column_list=None, replication_num=None, database_name=None, set_null=False, keys_desc=None, bitmap_index_list=None, dynamic_partition_info=None, replication_allocation=None, light_schema_change=None, enable_unique_key_merge_on_write=None): """ Create table Attributes: column_list: 由4元组(name, type, agg_type, default_value)组成的list, 后两项可省略需要注意的是key列指定默认值是agg_type设置为None Example: [("k1", "int"), ("k2", "char", None, ""), ("v", "date", "replace")] bitmap_index_list: 由3元组(index_name, column_name, index_type)组成的list Example:[("k1_index", "k1", "BITMAP")] partition_info: PartitionInfo对象 keys_desc: "AGGREGATE KEYS(k1)" dynamic_partition_info:创建动态分区用到的参数 replication_allocation:设置副本的资源组 enable_unique_key_merge_on_write:unique表是否开启merge-on-write数据更新模式，默认开启 deprecated: storage_type: distribution: added: 3种数据模型 aggregate，duplicate，unique """ database_name = database_name if database_name is not None else self.database_name # table name sql = 'CREATE TABLE %s.%s (' % (database_name, table_name) # columns key_columns = list() aggregate_flag = False for column in column_list: sql = '%s %s,' % (sql, util.column_to_sql(column, set_null)) if len(column) == 2 or column[2] is None: key_columns.append(column[0]) if len(column) > 2 and column[2] is not None and column[2].upper() in \ ['MAX', 'MIN', 'SUM', 'REPLACE', 'HLL_UNION', 'REPLACE_IF_NOT_NULL']: aggregate_flag = True if bitmap_index_list is not None: for bitmap_index in bitmap_index_list: sql = '%s %s,' % (sql, util.bitmap_index_to_sql(bitmap_index)) sql = '%s )' % sql.rstrip(',') if keys_desc is None: if aggregate_flag: keys_desc = 'AGGREGATE KEY(%s)' % ','.join(key_columns) else: keys_desc = '' sql = '%s %s' % (sql, keys_desc) # partition if partition_info: sql = '%s %s' % (sql, str(partition_info)) # distribution if distribution_info is None: distribution_info = DistributionInfo('HASH(%s)' % column_list[0][0], 5) elif isinstance(distribution_info, DistributionInfo) and \ distribution_info.distribution_type.upper() == 'RANDOM': distribution_info.distribution_type = 'HASH(%s)' % column_list[0][0] elif isinstance(distribution_info, str): distribution_info.replace(' RANDOM', 'HASH(%s)' % column_list[0][0]) distribution_info.replace(' random', 'HASH(%s)' % column_list[0][0]) else: pass sql = '%s %s' % (sql, str(distribution_info)) # properties sql = '%s PROPERTIES (' % (sql) if storage_medium is not None: sql = '%s "storage_medium"="%s",' % (sql, storage_medium) if storage_cooldown_time is not None: sql = '%s "storage_cooldown_time"="%s",' % (sql, storage_cooldown_time) if bloom_filter_column_list is not None: sql = '%s "bloom_filter_columns"="%s",' % (sql, ",".join(bloom_filter_column_list)) if replication_num is not None: sql = '%s "replication_num"="%s",' % (sql, replication_num) if dynamic_partition_info is not None: sql = '%s %s' % (sql, str(dynamic_partition_info)) if replication_allocation is not None: sql = '%s "replication_allocation"="%s"' % (sql, replication_allocation) if light_schema_change is not None: sql = '%s "light_schema_change"="%s"' % (sql, light_schema_change) if enable_unique_key_merge_on_write is not None: sql = '%s "enable_unique_key_merge_on_write"="%s"' % (sql, enable_unique_key_merge_on_write) if sql.endswith(' PROPERTIES ('): sql = sql.rstrip(' PROPERTIES (') else: sql = '%s )' % (sql.rstrip(',')) try: ret = self.execute(sql) except Exception as e: LOG.warning(L('CREATE TABLE fail.', table_name=table_name, msg=str(e))) if "Failed to find" in str(e) or \ "replication num should be less than the number of available backends" in str(e): alive_be = self.get_alive_backend_list() if len(alive_be) < 3: LOG.warning(L('SKIP: some backends are dead, create table failed, skip test case ')) raise pytest.skip("some backends are dead, create table failed, skip test case") else: raise e else: raise e if ret == (): LOG.info(L('CREATE TABLE succ.', database_name=database_name, table_name=table_name)) return True else: LOG.info(L('CREATE TABLE fail.', database_name=database_name, table_name=table_name)) return False def create_table_like(self, table_name, source_table_name, database_name=None, source_database_name=None, rollup_list=None, external=False, if_not_exists=False): """help create table like see more info""" sql = 'CREATE {external}TABLE {if_not_exists}{database_name}{table_name} LIKE ' \ '{source_database_name}{source_table_name}{with_rollup}' external = 'EXTERNAL' if external else '' if_not_exists = 'IF NOT EXISTS ' if if_not_exists else '' database_name = database_name + '.' if database_name else '' with_rollup = ' ROLLUP (%s)' % ','.join(rollup_list) if rollup_list else '' source_database_name = source_database_name + '.' if source_database_name else '' sql = sql.format(external=external, if_not_exists=if_not_exists, database_name=database_name, table_name=table_name, source_database_name=source_database_name, source_table_name=source_table_name, with_rollup=with_rollup) ret = self.execute(sql) return ret == () def create_rollup_table(self, table_name, rollup_table_name, column_name_list, storage_type=None, database_name=None, base_index_name=None, is_wait=False, force_alter=False, cluster_name=None, after_column_name_list=''): """ Create a rollup table todo:storage_type """ database_name = database_name if database_name is not None else self.database_name sql = 'ALTER TABLE %s.%s ADD ROLLUP %s (%s) %s' % (database_name, table_name, rollup_table_name, ','.join(column_name_list), after_column_name_list) if base_index_name: sql = "%s FROM %s" % (sql, base_index_name) if storage_type or force_alter: sql = '%s PROPERTIES(' % sql if storage_type: sql = '%s "storage_type"="column",' % sql if force_alter: sql = '%s "force_alter"="true",' % sql sql = '%s)' % sql.rstrip(',') ret = self.execute(sql) if ret != (): LOG.info(L("CREATE ROLLUP TABLE fail.", database_name=database_name, table_name=table_name, rollup_table_name=rollup_table_name)) return False ret = True if is_wait: ret = self.wait_table_rollup_job(table_name, cluster_name=cluster_name, database_name=database_name) return ret LOG.info(L("CREATE ROLLUP TABLE succ.", database_name=database_name, table_name=table_name, rollup_table_name=rollup_table_name)) return ret def cancel_rollup(self, table_name, database_name=None): """ 取消rollup """ database_name = database_name if database_name is not None else self.database_name sql = 'CANCEL ALTER TABLE ROLLUP FROM %s.%s' % (database_name, table_name) ret = self.execute(sql) if ret == (): LOG.info(L("CANCEL ALTER ROLLUP succ.", database_name=database_name, table_name=table_name)) return True else: LOG.info(L("CANCEL ALTER ROLLUP fail.", database_name=database_name, table_name=table_name)) return False def create_materialized_view(self, table_name, materialized_view_name, view_sql, database_name=None, is_wait=False): """ create_materialized_view :param table_name: :param materialized_view_name: :param view_sql: :param database_name: :param is_wait: :return: """ database_name = database_name if database_name is not None else self.database_name sql = 'CREATE MATERIALIZED VIEW %s AS %s' % (materialized_view_name, view_sql) # before do it sleep 1s time.sleep(1) ret = self.execute(sql) if ret != (): LOG.info(L("CREATE MATERIALIZED VIEW fail.", database_name=database_name, materialized_view_name=materialized_view_name)) return False ret = True if is_wait: ret = self.wait_table_rollup_job(table_name, database_name=database_name) return ret LOG.info(L("CREATE MATERIALIZED VIEW succ.", database_name=database_name, table_name=table_name, materialized_view_name=materialized_view_name, )) return ret def drop_materialized_view(self, database_name, table_name, view_name): """ drop_materialized_view 目前没有delete功能，rd开发中，采用alter table的方式删除 :param database_name: :param table_name: :param view_name: :return: """ sql = 'DROP MATERIALIZED VIEW IF EXISTS %s ON %s.%s' % (view_name, database_name, table_name) ret = self.execute(sql) return ret def get_index_list(self, table_name, database_name=None): """ 获取table的所有index """ if not table_name: return None ret = self.desc_table(table_name, database_name, is_all=True) idx_list = util.get_attr(ret, palo_job.DescInfoAll.IndexName) while '' in idx_list: idx_list.remove('') return idx_list def get_index(self, table_name, index_name=None, database_name=None): """ 获取table的指定名字的Index，如无指定，返回默认创建的Index info """ if not table_name: return None if index_name is None: index_name = table_name idx_list = self.get_index_list(table_name, database_name) if index_name in idx_list: return index_name else: LOG.info(L('can not get index from table', index_name=index_name, table_name=table_name)) return None def get_index_schema(self, table_name, index_name=None, database_name=None): """ Get table schema """ if index_name is None: return self.desc_table(table_name, database_name, is_all=False) ret = self.desc_table(table_name, database_name, is_all=True) record = False return_ret = list() for item in ret: desc_column = palo_job.DescInfoAll(item) if record is False and desc_column.get_index_name() != index_name: continue elif desc_column.get_index_name() == index_name: record = True return_ret.append((desc_column.get_field(), desc_column.get_type(), desc_column.get_null(), desc_column.get_key(), desc_column.get_default(), desc_column.get_extra())) elif record is True: return_ret.append((desc_column.get_field(), desc_column.get_type(), desc_column.get_null(), desc_column.get_key(), desc_column.get_default(), desc_column.get_extra())) else: LOG.info(L('get index schema error')) return tuple(return_ret) def set_time_zone(self, zone, global_var=False): """设置系统的时区 zone:要设置的时区,是否要global参数 """ if global_var: sql = "set global time_zone = '%s'" % zone else: sql = "set time_zone = '%s'" % zone LOG.info(L('palo sql', sql=sql)) palo_res = self.execute(sql) def set_sql_mode(self, sql_mode=None): """set_sql_mode""" sql_mode = "PIPES_AS_CONCAT" if sql_mode is None else sql_mode sql = "set sql_mode = %s" % sql_mode LOG.info(L("palo sql", sql=sql)) res = self.execute(sql) return res def get_sql_mode(self): """get sql mode""" sql = "select @@sql_mode" LOG.info(L("palo sql", sql=sql)) p_res = self.execute(sql) return p_res def wait_load_job(self, load_label, database_name=None, cluster_name=None, timeout=1800): """ wait load job """ database_name = database_name if database_name is not None else self.database_name LOG.debug(L("wait load job.", load_label=load_label, database_name=database_name)) retry_times = 0 sql = 'SHOW LOAD FROM %s WHERE LABEL="%s"' % (database_name, load_label) while retry_times < 10 and timeout > 0: time.sleep(1) timeout -= 1 load_job = self.execute(sql) if len(load_job) == 0: retry_times += 1 continue state = palo_job.LoadJob(load_job[-1]).get_state() if state.upper() != "FINISHED" and state.upper() != "CANCELLED": continue elif state.upper() == "FINISHED": return True else: LOG.info(L("LOAD FAILED.", database_name=database_name, msg=palo_job.LoadJob(load_job[-1]).get_errormsg(), url=palo_job.LoadJob(load_job[-1]).get_url())) return False else: LOG.info(L("LOAD LABEL NOT EXIST", load_label=load_label, database_name=database_name)) return False def bulk_load(self, table_name, load_label, data_file, max_filter_ratio=None, column_name_list=None, timeout=None, database_name=None, host=None, port=None, user=None, password=None, is_wait=False, cluster_name=None, hll_column_list=None, column_separator=None, backend_id=None): """ Args: table_name: string, table name load_label: string, load label data_file: string, 导入的文件 max_filter_ratio: float, max_filter_ratio column_name_list: list, column name list timeout: int, timeout database_name: string, database name host: string, fe host port: int, fe http port user: string, fe user password: string, fe password is_wait: True/False, is wait cluster_name: string, cluster name hll_column_list: list, like ['hll_column1,k1', 'hll_column,k2'] backend_id兼容以前的代码 Returns: True: mini load success False: mini load Fail """ database_name = database_name if database_name is not None else self.database_name host = self.host if host is None else host port = self.http_port if port is None else port user = self.user if user is None else user password = self.password if password is None else password url = 'http://%s:%s/api/%s/%s/_load?label=%s' % (host, port, database_name, table_name, load_label) if max_filter_ratio: url = '%s&max_filter_ratio=%s' % (url, max_filter_ratio) if column_name_list: url = '%s&columns=%s' % (url, ','.join(column_name_list)) if hll_column_list: url = '%s&hll=%s' % (url, ':'.join(hll_column_list)) if timeout: url = '%s&timeout=%s' % (url, timeout) if column_separator: url = '%s&column_separator=%s' % (url, column_separator) cmd = 'curl --location-trusted -u %s:%s -T %s %s' % (user, password, data_file, url) print(cmd) LOG.info(L('bulk multi load', cmd=cmd)) file = open(data_file) c = pycurl.Curl() buf = BytesIO() c.setopt(c.URL, url) c.setopt(pycurl.WRITEFUNCTION, buf.write) # basic认证 c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) c.setopt(pycurl.USERNAME, user) c.setopt(pycurl.PASSWORD, password) # 上传文件 c.setopt(pycurl.UPLOAD, 1) c.setopt(pycurl.READDATA, file) # 重定向, --location-trusted c.setopt(pycurl.UNRESTRICTED_AUTH, 1) c.setopt(pycurl.FOLLOWLOCATION, True) LOG.info(L("BULK LOAD.", url=url, file_name=data_file)) msg = None try: c.perform() msg = buf.getvalue() LOG.info(L("BULK LOAD ret.", ret=msg)) print(c.getinfo(c.HTTP_CODE)) # if c.getinfo(c.HTTP_CODE) == 200: if c.getinfo(c.HTTP_CODE): ret = json.loads(msg) status = ret.get('status') if status == 'Success': if is_wait: r = self.wait_load_job(load_label, database_name, cluster_name=cluster_name) return r else: return True elif status == 'Fail': return False else: return False except Exception as e: LOG.info(L("BULK LOAD failed.", err=str(e), msg=msg)) return False def batch_load(self, load_label, load_data_list, max_filter_ratio=None, timeout=None, hadoop_info=None, by_load_cluster=None, property_list=None, database_name=None, is_wait=False, cluster_name=None, broker=None, strict_mode=None, timezone=None, temp_partition=None): """ Load data Attributes: load_data_list: LoadDataInfo对象或list """ database_name = self.database_name if database_name is None else database_name data_list = list() if isinstance(load_data_list, LoadDataInfo): data_list.append(str(load_data_list)) elif isinstance(load_data_list, list): for data in load_data_list: data_list.append(str(data)) else: raise PaloClientException('Load data list should be list or LoadDataInfo', load_data_list=load_data_list) sql = 'LOAD LABEL %s.%s (%s)' % (database_name, load_label, ', '.join(data_list)) if by_load_cluster is not None: sql = '%s BY "%s"' % (sql, by_load_cluster) if broker is not None: sql = '%s %s' % (sql, str(broker)) sql = '%s %s' % (sql, 'PROPERTIES(') if max_filter_ratio is not None: sql = '%s "max_filter_ratio"="%s",' % (sql, max_filter_ratio) if timeout is not None: sql = '%s "timeout"="%d",' % (sql, timeout) if strict_mode is not None: sql = '%s "strict_mode"="%s",' % (sql, strict_mode) if timezone is not None: sql = '%s "timezone"="%s",' % (sql, timezone) if hadoop_info is not None: sql = '%s %s' % (sql, str(hadoop_info)) if property_list: sql = sql + ', '.join(property_list) + ',' if sql.endswith(','): sql = sql.rstrip(',') sql = '%s %s' % (sql, ')') else: sql = sql.rstrip('PROPERTIES(') ret = self.execute(sql) if ret != (): LOG.info(L('LOAD fail.', database_name=database_name, load_label=load_label)) return False if is_wait: ret = self.wait_load_job(load_label, database_name, cluster_name=cluster_name) if not ret: LOG.info(L('LOAD fail.', database_name=database_name, load_label=load_label)) return False LOG.info(L('LOAD succ.', database_name=database_name, load_label=load_label)) return True def cancel_load(self, load_label, database_name=None): """ 取消导入任务 """ database_name = self.database_name if database_name is None else database_name sql = 'CANCEL LOAD FROM %s WHERE LABEL = "%s"' % (database_name, load_label) ret = self.execute(sql) if ret == (): LOG.info(L("CANCEL LOAD succ.", database_name=database_name, label=load_label)) return True else: LOG.info(L("CANCEL LOAD fail.", database_name=database_name, label=load_label)) return False def get_quota(self, database_name, cluster_name=None): """ get quota """ if cluster_name: database_name = '%s:%s' % (cluster_name, database_name) sql = "SHOW PROC '/dbs'" result = self.execute(sql) for info in result: if info[1] == database_name: return info[3] LOG.warning(L("Get quota fail.", database_name=database_name)) return None def alter_database(self, database_name, quota): """ alter database """ sql = "ALTER DATABASE %s SET DATA QUOTA %d" % (database_name, quota) result = self.execute(sql) if result != (): LOG.error(L("ALTER DATABASE fail.", database_name=database_name, quota=quota)) return False return True def drop_database(self, database_name=None, force=None): """ 删除数据库 Parameters： database_name：如果为None，将删除默认的database. Type: str. Returns: False：数据库删除失败 True：数据库删除成功 Raises: PaloClientException: 数据库删除异常 """ database_name = self.database_name if database_name is None else database_name if self.database_name == database_name: self.database_name = None sql = "DROP DATABASE %s" % database_name if force is True: sql = "DROP DATABASE %s FORCE" % database_name self.execute(sql) database_name_list = self.get_database_list() if database_name in database_name_list: LOG.warning(L("DROP DATABASE fail.", database_name=database_name)) return False else: LOG.info(L("DROP DATABASE succ.", database_name=database_name)) return True def clean(self, database_name=None): """ 清除所有数据 Parameters： database_name：默认删除所有database。Type: str. Returns: None """ if database_name is None: database_name_list = self.get_database_list() for database_name in database_name_list: if database_name.find("information_schema") == -1: self.clean(database_name) else: database_name_list = self.get_database_list() while database_name in database_name_list: try: self.drop_database(database_name, force=True) except PaloClientException: pass database_name_list = self.get_database_list() def drop_table(self, table_name, database_name=None, cluster_name=None, if_exist=False): """删除table family Parameters： database_name：如果为None，将删除默认的database. [Type str] table_family_name: table family name. [Type str] Returns: 目前无返回值 Raises: 没有捕获，可能抛出 """ database_name = self.database_name if database_name is None else database_name if not if_exist: sql = 'DROP TABLE' else: sql = 'DROP TABLE IF EXISTS' sql = "%s %s.%s" % (sql, database_name, table_name) self.execute(sql) return True def drop_rollup_table(self, table_name, rollup_table_name, database_name=None): """删除rollup table Parameters： database_name：如果为None，将删除默认的database. [Type str] table_family_name: table family name. [Type str] Returns: 目前无返回值 Raises: 没有捕获，可能抛出 """ database_name = self.database_name if database_name is None else database_name sql = "ALTER TABLE %s.%s DROP ROLLUP %s" % \ (database_name, table_name, rollup_table_name) ret = self.execute(sql) return ret == () def select_all(self, table_name, database_name=None): """ select all """ database_name = self.database_name if database_name is None else database_name sql = "SELECT * FROM %s.%s" % (database_name, table_name) result = self.execute(sql) return result def query(self, sql): """ query """ result = self.execute(sql) return result def get_load_job_state(self, label, database_name=None, cluster_name=None): """ get load job state """ load_job_list = self.get_load_job_list(database_name=database_name, \ cluster_name=cluster_name) for load_job in load_job_list: job = palo_job.LoadJob(load_job) if job.get_label() == label: return job.get_state() return None def get_unfinish_load_job_list(self, database_name=None, cluster_name=None): """ 获取所有未完成的导入任务，即：状态为pending、etl、loading的任务 """ load_job_list = self.get_load_job_list( \ database_name=database_name, cluster_name=cluster_name) result = list() for load_job in load_job_list: job = palo_job.LoadJob(load_job) if job.get_state() != "FINISHED" and job.get_state() != "CANCELLED": result.append(load_job) return result def get_load_job_list(self, state=None, database_name=None, cluster_name=None): """ 获取指定状态的导入任务信息, according job state get load job """ database_name = self.database_name if database_name is None else database_name sql = 'SHOW LOAD FROM %s' % database_name job_list = self.__execute_and_rebuild_meta_class(sql, palo_job.LoadJob) result = list() if state: state = state.upper() for job in job_list: if palo_job.LoadJob(job).get_state() == state: result.append(job) return result else: return job_list def get_load_job(self, label, database_name=None, cluster_name=None): """ get load job, according label get load job """ load_job_list = self.get_load_job_list(database_name=database_name, \ cluster_name=cluster_name) for load_job in load_job_list: if palo_job.LoadJob(load_job).get_label() == label: return load_job return None def get_delete_job_list(self, state=None, database_name=None, cluster_name=None): """ 获取指定状态的delete job信息 """ database_name = self.database_name if database_name is None else database_name if not cluster_name: cluster_name = self.cluster_name if cluster_name: database_name = '%s:%s' % (cluster_name, database_name) sql = 'SHOW DELETE FROM %s' % database_name job_list = self.execute(sql) result = list() if state: state = state.upper() for job in job_list: if palo_job.DeleteJob(job).get_state() == state: result.append(job) return result else: return job_list def wait_table_rollup_job(self, table_name, database_name=None, cluster_name=None, state='FINISHED', timeout=2000): """ 等待rollup完成 """ time.sleep(5) database_name = self.database_name if database_name is None else database_name retry_times = 10 while timeout > 0: job_list = self.get_table_rollup_job_list(table_name, database_name, cluster_name=cluster_name) if not job_list: retry_times -= 1 if retry_times == 0: LOG.info(L("CANNOT GET ROLLUP JOB.", database_name=database_name)) break time.sleep(1) continue last_job_state = palo_job.RollupJob(job_list[-1]).get_state() if last_job_state == state: LOG.info(L("GET ROLLUP JOB STATE.", database_name=database_name, state=palo_job.RollupJob(job_list[-1]).get_state())) return True if last_job_state == 'CANCELLED': LOG.info(L("ROLLUP JOB CANCELLED.", database_name=database_name, msg=palo_job.RollupJob(job_list[-1]).get_msg())) return False if state != 'FINISHED' and last_job_state == 'FINISHED': LOG.info(L("ROLLUP JOB FINISHED.", database_name=database_name)) return False time.sleep(3) timeout -= 3 LOG.warning(L("WAIT ROLLUP JOB TIMEOUT.", database_name=database_name)) return False def wait_table_load_job(self, database_name=None, timeout=1800): """等待db的所有load任务完成""" database_name = self.database_name if database_name is None else database_name flag = False while timeout > 0: flag = True load_job_list = self.execute('SHOW LOAD FROM %s' % database_name) for job in load_job_list: LOG.info(L("LOAD JOB STATE.", database_name=database_name, state=job[palo_job.LoadJob.State])) if job[palo_job.LoadJob.State] != 'CANCELLED' and \ job[palo_job.LoadJob.State] != 'FINISHED': LOG.info(L("LOAD JOB RUNNING.", database_name=database_name, state=job[palo_job.LoadJob.State])) flag = False break time.sleep(3) timeout -= 3 if flag: LOG.info(L("LOAD JOB FINISHED.", database_name=database_name)) break LOG.info(L("WAIT LOAD JOB FINISHED.", database_name=database_name)) def get_table_rollup_job_list(self, table_name, database_name=None, cluster_name=None): """ 获取指定table family的rollup任务信息 """ database_name = self.database_name if database_name is None else database_name sql = 'SHOW ALTER TABLE ROLLUP FROM %s WHERE TableName = "%s"' % (database_name, table_name) database_rollup_job_list = self.__execute_and_rebuild_meta_class(sql, palo_job.RollupJob) table_rollup_job_list = [] for rollup_job in database_rollup_job_list: if palo_job.RollupJob(rollup_job).get_table_name() == table_name: table_rollup_job_list.append(rollup_job) return table_rollup_job_list def get_database_rollup_job_list(self, database_name=None, cluster_name=None): """ 获取database的所有rollup job信息 """ database_name = self.database_name if database_name is None else database_name if cluster_name: database = '%s:%s' % (cluster_name, database_name) sql = "SHOW PROC '/jobs/%s/rollup'" % self.get_database_id(database_name) return self.execute(sql) def get_database_list(self, cluster_name=None): """ 显示所有的database name list Parameters： None Returns： database name list Raises： PaloClientException：获取数据异常 """ sql = r"SHOW DATABASES" result = self.execute(sql) database_name_list = [name[0] for name in result] return database_name_list def get_partition_list(self, table_name, database_name=None, cluster_name=None): """ get table families """ database_name = self.database_name if database_name is None else database_name if cluster_name: database_name = '%s:%s' % (cluster_name, database_name) sql = "SHOW PARTITIONS FROM %s.%s" % (database_name, table_name) result = self.__execute_and_rebuild_meta_class(sql, palo_job.PartitionInfo) return result def get_partition(self, table_name, partition_name, database_name=None, cluster_name=None): """ 获取指定的table family """ partition_list = self.get_partition_list(table_name, database_name, \ cluster_name=cluster_name) for partition in partition_list: if partition[palo_job.PartitionInfo.PartitionName] == partition_name: return partition return None def get_partition_id(self, table_name, partition_name, database_name=None): """ get table family id. Parameters： database_name table_family_name Returns: None：如果table family不存在 """ partition = self.get_partition(table_name, \ partition_name, database_name) if partition: return partition[palo_job.PartitionInfo.PartitionId] else: return None def get_partition_name_by_id(self, table_name, partition_id, \ database_name=None, cluster_name=None): """get partition name by id """ partition_list = self.get_partition_list(table_name, \ database_name, cluster_name=cluster_name) for partition in partition_list: if partition[palo_job.PartitionInfo.PartitionId] == partition_id: return partition[palo_job.PartitionInfo.PartitionName] return None def get_partition_version(self, table_name, \ partition_name, database_name=None, cluster_name=None): """ 获取table family的version号 """ partition = self.get_partition(table_name, \ partition_name, database_name, cluster_name=cluster_name) if partition: return partition[palo_job.PartitionInfo.VisibleVersion] else: return None def get_partition_storage_medium(self, table_name, partition_name, \ database_name=None, cluster_name=None): """ get table family id. Parameters： database_name table_family_name Returns: None：如果table family不存在 """ partition = self.get_partition(table_name, \ partition_name, database_name, cluster_name=cluster_name) if partition: return partition[palo_job.PartitionInfo.StorageMedium] else: return None def get_partition_cooldown_time(self, table_name, partition_name, \ database_name=None, cluster_name=None): """ get table family id. Parameters： database_name table_family_name Returns: None：如果table family不存在 """ partition = self.get_partition(table_name, partition_name, database_name, cluster_name=cluster_name) if partition: return partition[palo_job.PartitionInfo.CooldownTime] else: return None def get_partition_replication_num(self, table_name, partition_name, database_name=None, cluster_name=None): """ get table family id. Parameters： database_name table_family_name Returns: None：如果table family不存在 """ partition = self.get_partition(table_name, partition_name, database_name, cluster_name=cluster_name) if partition: return partition[palo_job.PartitionInfo.ReplicationNum] else: return None def get_partition_buckets(self, table_name, partition_name, database_name=None, cluster_name=None): """ get table family id. Parameters： database_name table_family_name Returns: None：如果table family不存在 """ partition = self.get_partition(table_name, partition_name, database_name, cluster_name=cluster_name) if partition: return partition[palo_job.PartitionInfo.Buckets] else: return None def delete(self, table_name, delete_conditions, partition_name=None, database_name=None, is_wait=False): """ 按条件删除指定 table family 中的数据 Parameters: delete_conditions：可以是删除条件组成的list，每个删除条件是一个三元组，如下。也可以是字符串由列名，比较运算符和值三项组成。如： [("k1", "=", "0"), ("k2", "<=", "10")] Returns: True：执行成功 False：执行失败 Raises： """ database_name = self.database_name if database_name is None else database_name if not partition_name: sql = "DELETE FROM %s.%s WHERE" % (database_name, table_name) else: sql = "DELETE FROM %s.%s PARTITION %s WHERE" % (database_name, table_name, partition_name) if isinstance(delete_conditions, list): for where_condition in delete_conditions: # 对value项加上引号, 防止datetime因空格分隔造成错误 where_condition = list(where_condition) where_condition[2] = "\"%s\"" % where_condition[2] sql = "%s %s AND" % (sql, " ".join(where_condition)) sql = sql[:-4] elif isinstance(delete_conditions, str): sql = "%s %s" % (sql, delete_conditions) # 同步的，返回时，已删除，未返回时，其他连接show delete状态为DELETING状态 time.sleep(1) ret = self.execute(sql) LOG.info(L("DELETE EXECUTE SUCC", ret=ret)) if is_wait is True: if database_name is not None: line = 'SHOW DELETE FROM %s' % database_name else: line = 'SHOW DELETE' delete_job = self.execute(line) for job in delete_job: if job[4] == 'FINISHED': pass else: time.sleep(1) return ret == () def show_delete(self, database_name=None): """show delete """ if not database_name: database_name = self.database_name sql = 'SHOW DELETE FROM %s' % database_name ret = self.execute(sql) return ret def schema_change(self, table_family_name, add_column_list=None, drop_column_list=None, modify_column_list=None, order_column_list=None, bloom_filter_column_list=None, database_name=None, colocate_with_list=None, distribution_type=None, is_wait=False, cluster_name=None, comment=None, replication_allocation=None): """schema change Parameters: table_family_name: rollup_table_name: add_column_list: 新加列，每个元素为一个新加列 "列名类型聚合方法 default '默认值' after '列名' in 'rollup表名'" e.g. ["addc1 int default '1'", \ "addc2 float sum default '2.3'", \ "addc3 int default '2' after k1", \ "addc1 int default '1' in 'rollup_table'"] drop_column_list: 删除列，每个元素为一个删除的列名 e.g. ["k1", "v"] modify_column_list: 修改列属性，每个元素为一个列, 参考add_column_list order_column_list: 调整列顺序，每个元素为一个删除的列名 bloom_filter_column_list: database_name: colocate_with_list: distribution_type: is_wait: cluster_name: comment:表注释 replication_allocation:副本标签 """ database_name = self.database_name if database_name is None else database_name sql = "ALTER TABLE %s.%s" % (database_name, table_family_name) if add_column_list: if len(add_column_list) == 1: sql = "%s ADD COLUMN %s" % (sql, ", ".join(add_column_list)) else: sql = "%s ADD COLUMN (%s)" % (sql, ", ".join(add_column_list)) if drop_column_list: sql = "%s DROP COLUMN %s" % (sql, ", DROP COLUMN ".join(drop_column_list)) if order_column_list: if len(order_column_list) == 1: sql = "%s ORDER BY %s" % (sql, ", ".join(order_column_list)) else: sql = "%s ORDER BY (%s)" % (sql, ", ".join(order_column_list)) if modify_column_list: sql = "%s MODIFY COLUMN %s" % (sql, ", MODIFY COLUMN ".join(modify_column_list)) if bloom_filter_column_list: if add_column_list is None and drop_column_list is None \ and modify_column_list is None and order_column_list is None: sql = '%s SET ("bloom_filter_columns"="%s")' % (sql, ",".join(bloom_filter_column_list)) else: sql = '%s PROPERTIES ("bloom_filter_columns"="%s")' % (sql, ",".join(bloom_filter_column_list)) if colocate_with_list: sql = '%s SET ("colocate_with"="%s")' % (sql, ",".join(colocate_with_list)) if distribution_type: sql = '%s SET ("distribution_type"="%s")' % (sql, ",".join(distribution_type)) if comment: sql = '%s MODIFY comment "%s"' % (sql, comment) if replication_allocation: sql = '%s SET ("replication_allocation"="%s")' % (sql, replication_allocation) result = self.execute(sql) if result != (): LOG.info(L("SCHEMA CHANGE fail.", database_name=database_name, table_family_name=table_family_name)) return False if is_wait: LOG.info(L("wait for SCHEMA CHANGE.", database_name=database_name, table_family_name=table_family_name)) if not self.wait_table_schema_change_job(table_family_name, cluster_name=cluster_name): LOG.info(L("SCHEMA CHANGE fail.", database_name=database_name, table_family_name=table_family_name)) return False return True def schema_change_add_column(self, table_name, column_list, after_column_name=None, to_table_name=None, force_alter=False, database_name=None, is_wait_job=False, is_wait_delete_old_schema=False, cluster_name=None, set_null=False): """ 增加列 column_list: column_name, column_type, aggtype, default_value same to function create table family """ database_name = self.database_name if database_name is None else database_name sql = 'ALTER TABLE %s.%s ADD COLUMN' % (database_name, table_name) if len(column_list) > 1: sql = '%s(' % sql for column in column_list: sql = '%s %s, ' % (sql, util.column_to_sql(column, set_null)) sql = sql.rstrip(', ') if len(column_list) > 1: sql = '%s)' % sql if after_column_name: if after_column_name == 'FIRST': sql = '%s %s' % (sql, after_column_name) else: sql = '%s AFTER %s' % (sql, after_column_name) if to_table_name: sql = '%s TO %s' % (sql, to_table_name) if force_alter: sql = '%s PROPERTIES("force_alter"="true")' % sql result = self.execute(sql) if result != (): LOG.info(L("SCHEMA CHANGE fail.", database_name=database_name, table_name=table_name)) return False if is_wait_job: if not self.wait_table_schema_change_job(table_name, cluster_name=cluster_name): return False return True def schema_change_drop_column(self, table_name, column_name_list, from_table_name=None, force_alter=False, database_name=None, is_wait_job=False, is_wait_delete_old_schema=False, cluster_name=None): """ 删除列 column_name_list: ['k1', 'v1', 'v3'] """ database_name = self.database_name if database_name is None else database_name sql = 'ALTER TABLE %s.%s' % (database_name, table_name) from_table_sql = '' if from_table_name: from_table_sql = 'FROM %s' % (from_table_name) force_alter_sql = '' if force_alter: force_alter_sql = 'PROPERTIES("force_alter"="true")' for column in column_name_list: sql = '%s DROP COLUMN %s %s %s, ' % (sql, column, from_table_sql, force_alter_sql) sql = sql.rstrip(', ') result = self.execute(sql) if result != (): LOG.info(L("SCHEMA CHANGE fail.", database_name=database_name, table_name=table_name)) return False if is_wait_job: if not self.wait_table_schema_change_job(table_name, cluster_name=cluster_name): return False return True def schema_change_order_column(self, table_name, column_name_list, from_table_name=None, force_alter=False, database_name=None, is_wait_job=False, is_wait_delete_old_schema=False, cluster_name=None): """ 重新排序 """ database_name = self.database_name if database_name is None else database_name sql = 'ALTER TABLE %s.%s' % (database_name, table_name) from_table_sql = '' if from_table_name: from_table_sql = 'FROM %s' % (from_table_name) force_alter_sql = '' if force_alter: force_alter_sql = 'PROPERTIES("force_alter"="true")' sql = '%s ORDER BY (%s)' % (sql, ', '.join(column_name_list)) sql = '%s %s %s' % (sql, from_table_sql, force_alter_sql) result = self.execute(sql) if result != (): LOG.info(L("SCHEMA CHANGE fail.", database_name=database_name, table_name=table_name)) return False if is_wait_job: if not self.wait_table_schema_change_job(table_name, cluster_name=cluster_name): return False return True def schema_change_modify_column(self, table_name, column_name, column_type, after_column_name=None, from_table_name=None, force_alter=False, database_name=None, is_wait_job=False, is_wait_delete_old_schema=False, cluster_name=None, aggtype='', column_info=""): """ 修改列类型 """ database_name = self.database_name if database_name is None else database_name schema = self.desc_table(table_name, database_name=database_name) if not aggtype: for field in schema: if field[0] == column_name and field[3] == 'false': aggtype = field[5].split(',')[0] break if aggtype == '-' or aggtype == 'NONE': aggtype = '' sql = 'ALTER TABLE %s.%s MODIFY COLUMN %s %s %s %s' % (database_name, table_name, column_name, column_type, aggtype, column_info) if after_column_name: sql = '%s AFTER %s' % (sql, after_column_name) if from_table_name: sql = '%s FROM %s' % (sql, from_table_name) if force_alter: sql = '%s PROPERTIES("force_alter"="true")' % sql result = self.execute(sql) if result != (): LOG.info(L("SCHEMA CHANGE fail.", database_name=database_name, table_name=table_name)) return False if is_wait_job: if not self.wait_table_schema_change_job(table_name, cluster_name=cluster_name): return False return True def wait_table_schema_change_job(self, table_name, database_name=None, cluster_name=None, state="FINISHED", timeout=1200): """ 等待schema change完成 """ database_name = self.database_name if database_name is None else database_name try_times = 0 while try_times < 120 and timeout > 0: time.sleep(3) timeout -= 3 schema_change_job_list = self.get_table_schema_change_job_list( table_name, database_name, cluster_name=cluster_name) if not schema_change_job_list or len(schema_change_job_list) == 0: try_times += 1 continue last_job_state = palo_job.SchemaChangeJob(schema_change_job_list[-1]).get_state() LOG.info(L("GET LAST SCHEMA CHANGE JOB STATE", state=last_job_state)) if last_job_state == state: LOG.info(L("GET SCHEMA CHANGE JOB STATE.", database_name=database_name, state=palo_job.SchemaChangeJob(schema_change_job_list[-1]).get_state())) return True if last_job_state == 'CANCELLED' and state != 'CANCELLED': LOG.info(L("SCHEMA CHANGE fail.", state='CANCELLED', msg=palo_job.SchemaChangeJob(schema_change_job_list[-1]).get_msg())) return False if state != 'FINISHED' and last_job_state == 'FINISHED': LOG.info(L("SCHEMA CHANGE FINISHED.", state='FINISHED')) return False LOG.warning(L("WAIT SCHEMA CHANGE TIMEOUT.", database_name=database_name)) return False def get_table_schema_change_job_list(self, table_name, database_name=None, cluster_name=None): """ 获取指定table的所有schema change job信息 """ database_name = self.database_name if database_name is None else database_name sql = 'SHOW ALTER TABLE COLUMN FROM %s' % database_name database_schema_change_job_list = self.__execute_and_rebuild_meta_class(sql, palo_job.SchemaChangeJob) table_schema_change_job_list = [] for schema_change_job in database_schema_change_job_list: if palo_job.SchemaChangeJob(schema_change_job).get_table_name() == table_name: table_schema_change_job_list.append(schema_change_job) return table_schema_change_job_list def cancel_schema_change(self, table_name, database_name=None): """ 取消rollup """ database_name = self.database_name if database_name is None else database_name sql = 'CANCEL ALTER TABLE COLUMN FROM %s.%s' % (database_name, table_name) ret = self.execute(sql) if ret == (): LOG.info(L("CANCEL SCHEMA CHANGE Succ.", database_name=database_name, table_name=table_name)) return True else: LOG.info(L("CANCEL SCHEMA CHANGE Fail.", database_name=database_name, table_name=table_name)) return False def add_partition(self, table_name, partition_name, value, distribute_type=None, bucket_num=None, storage_medium=None, storage_cooldown_time=None, database_name=None, partition_type=None): """ 增加分区增加分区的时候，只支持默认的表的分桶方式，不支持其他新的分桶方式,可修改bucket数量 Args: table_name: table name partition_name: new partition name value: str or tuple, like: 'k1' or ('k1', 'k2'), (('1','2'),('3','4')) distribute_type: only hash, do not support random bucket_num: num storage_medium: storage_cooldown_time: database_name: Returns: True/except """ database_name = self.database_name if database_name is None else database_name predicate = 'IN' if partition_type is not None and partition_type.upper() == 'LIST' else 'LESS THAN' if value != 'MAXVALUE': # 单列list or 多列range if isinstance(value, tuple) and isinstance(value[0], str): value = '(%s)' % ','.join('"{0}"'.format(v) for v in value) value = value.replace('"MAXVALUE"', 'MAXVALUE') # 多列list elif isinstance(value, tuple) and isinstance(value[0], tuple): in_val_list = [] for multi_col_value in value: in_val_list.append('(%s)' % ','.join('"{0}"'.format(v) for v in multi_col_value)) value = '(%s)' % ','.join(in_val_list) # 单列range else: value = '("%s")' % (value) sql = 'ALTER TABLE %s.%s ADD PARTITION %s VALUES %s %s' % ( database_name, table_name, partition_name, predicate, value) sql = '%s (' % (sql) if storage_medium is not None: sql = '%s "storage_medium"="%s",' % (sql, storage_medium) if storage_cooldown_time is not None: sql = '%s "storage_cooldown_time"="%s",' % (sql, storage_cooldown_time) if sql.endswith(' ('): sql = sql.rstrip(' (') else: sql = '%s )' % (sql.rstrip(',')) if distribute_type: if distribute_type.upper() == 'RANDOM': discol = self.execute('desc %s.%s' % (database_name, table_name))[0][0] sql = '%s DISTRIBUTED BY HASH(%s)' % (sql, discol) else: sql = '%s DISTRIBUTED BY %s' % (sql, distribute_type) if bucket_num: sql = '%s BUCKETS %d' % (sql, bucket_num) result = self.execute(sql) if result != (): LOG.info(L("ADD PARTITION fail.", database_name=database_name, table_name=table_name)) return False return True def modify_partition(self, table_name, partition_name=None, storage_medium=None, storage_cooldown_time=None, replication_num=None, database_name=None, **kwargs): """ 修改分区的 storage_medium、storage_cooldown_time 和 replication_num 三个属性。对于单分区表，partition_name 同表名如果partition_name为none则为修改全表属性 """ database_name = self.database_name if database_name is None else database_name sql = 'ALTER TABLE %s.%s' % (database_name, table_name) if partition_name is not None: if isinstance(partition_name, list): partition_name = '(' + ','.join(partition_name) + ')' sql = '%s MODIFY PARTITION %s' % (sql, partition_name) if storage_medium is not None: kwargs['storage_medium'] = storage_medium if storage_cooldown_time is not None: kwargs['storage_cooldown_time'] = storage_cooldown_time if replication_num is not None: kwargs['replication_num'] = replication_num property = '' for k, v in kwargs.items(): property += ' "%s" = "%s",' % (k, v) property_str = property.strip(',') sql = '%s SET(%s)' % (sql, property_str) result = self.execute(sql) if result != (): LOG.info(L("MODIFY PARTITION fail.", database_name=database_name, table_name=table_name)) return False return True def drop_partition(self, table_name, partition_name, database_name=None): """ 删除分区 """ database_name = self.database_name if database_name is None else database_name sql = 'ALTER TABLE %s.%s DROP PARTITION %s' % ( \ database_name, table_name, partition_name) try: self.execute(sql) except PaloException as e: LOG.error(L('', fe=str(self), error=e)) return False return True def add_temp_partition(self, table_name, partition_name, value, distribute_type=None, bucket_num=None, in_memory=None, replication_num=None, database_name=None, partition_type='RANGE'): """ 新建临时分区 :param table_name: :param partition_name: :param value: :param distribute_type: :param bucket_num: :param in_memory: :param replication_num: :param database_name: :return: """ database_name = self.database_name if database_name is None else database_name partitioninfo = PartitionInfo(partition_type=partition_type).get_partition_value(value) print(partitioninfo) sql = 'ALTER TABLE %s.%s ADD TEMPORARY PARTITION %s %s' % (database_name, table_name, partition_name, partitioninfo) sql = '%s (' % (sql) if in_memory is not None: sql = '%s "in_memory"="%s",' % (sql, in_memory) if replication_num is not None: sql = '%s "replication_num"="%s",' % (sql, replication_num) if sql.endswith(' ('): sql = sql.rstrip(' (') else: sql = '%s )' % (sql.rstrip(',')) if distribute_type: if distribute_type.upper() == 'RANDOM': discol = self.execute('desc %s.%s' % (database_name, table_name))[0][0] sql = '%s DISTRIBUTED BY HASH(%s)' % (sql, discol) else: sql = '%s DISTRIBUTED BY %s' % (sql, distribute_type) if bucket_num: sql = '%s BUCKETS %d' % (sql, bucket_num) result = self.execute(sql) if result != (): LOG.info(L("ADD TEMP PARTITION fail.", database_name=database_name, table_name=table_name)) return False return True def drop_temp_partition(self, database_name, table_name, partition_name): """ 删除临时分区 :param database_name: :param table_name: :param partition_name: :return: """ self.use(database_name) sql = 'ALTER TABLE %s DROP TEMPORARY PARTITION %s' % (table_name, partition_name) result = self.execute(sql) if result != (): LOG.info(L("DROP TEMP PARTITION fail.", database_name=database_name, table_name=table_name)) return False return True def modify_temp_partition(self, database_name, table_name, target_partition_list, temp_partition_list, strict_range=None, use_temp_partition_name=None): """ 修改临时分区 :param database_name: :param table_name: :param target_partition_list: :param temp_partition_list: :param strict_range: :param use_temp_partition_name: :return: """ self.use(database_name) target_partition = ','.join(target_partition_list) temp_partition = ','.join(temp_partition_list) sql = 'ALTER TABLE %s.%s REPLACE PARTITION (%s) WITH TEMPORARY PARTITION (%s)' % \ (database_name, table_name, target_partition, temp_partition) if strict_range is not None or use_temp_partition_name is not None: sql = "%s PROPERTIES (" % sql value_list = [] print(strict_range) if strict_range: value_list.append('"strict_range"="%s"' % strict_range) if use_temp_partition_name: value_list.append('"use_temp_partition_name"="%s"' % use_temp_partition_name) print(value_list) sql = "%s %s)" % (sql, ','.join(value_list)) result = self.execute(sql) if result != (): LOG.info(L("ALTER TEMP PARTITION fail.", database_name=database_name, table_name=table_name)) return False return True def create_user(self, user, password=None, is_superuser=False, default_role=None): """ create user """ sql = "CREATE USER '%s'" % user if password is not None: sql = "%s IDENTIFIED BY '%s'" % (sql, password) if is_superuser: sql = "%s SUPERUSER" % sql if default_role: sql = "%s DEFAULT ROLE '%s'" % (sql, default_role) result = self.execute(sql) if result != (): LOG.error(L("CREATE USER fail.", user=user, password=password, \ is_superuser=is_superuser)) return False return True def create_role(self, role_name): """ create role """ sql = 'CREATE ROLE %s' % role_name result = self.execute(sql) if result != (): LOG.error(L("CREATE ROLE fail.", sql=sql, msg=str(result))) return False return True def drop_role(self, role): """drop role""" sql = "DROP ROLE %s" % role result = self.execute(sql) if result != (): LOG.error(L("DROP ROLE fail", msg=str(result), role=role)) return False return True def drop_user(self, user, if_exists=True): """ drop user """ if if_exists: sql = "DROP USER IF EXISTS '%s'" % user else: sql = "DROP USER '%s'" % user result = self.execute(sql) if result != (): LOG.error(L("DROP USER fail.", user=user)) return False return True def clean_user(self, user): """clean user """ try: self.drop_user(user) except: pass def set_password(self, user=None, password=None): """ set password of user to password """ if user is None: user = self.user if password is None: password = '' sql = "SET PASSWORD FOR '%s' = PASSWORD('%s')" % (user, password) result = self.execute(sql) if result != (): LOG.error(L("SET PASSWORD fail.", user=user, password=password)) return False return True def alter_user(self, user, cpu_share=None, max_user_connections=None): """ alter user """ sql = "ALTER USER '%s'" % user if cpu_share is not None: sql = "%s MODIFY RESOURCE CPU_SHARE %d," % (sql, cpu_share) if max_user_connections is not None: sql = "%s MODIFY PROPERTY MAX_USER_CONNECTIONS %d," % (sql, max_user_connections) sql = sql.rstrip(",") result = self.execute(sql) if result != (): LOG.error(L("ALTER USER fail.", user=user, cpu_share=cpu_share, max_user_connections=max_user_connections)) return False return True def get_cpu_share(self, user): """ get cpu share """ result = self.show_resource(user) for info in result: if info[0] == user and info[1] == "CPU_SHARE": return int(info[2]) return None def show_resource(self, user): """ show resource """ sql = "SHOW RESOURCE LIKE '%s'" % user result = self.execute(sql) if result == (): LOG.warning(L("SHOW RESOURCE fail.", user=user)) return result def grant(self, user, privilege_list, database='', table='', catalog='', resource=None, is_role=False, identity='%'): """ grant GRANT privilege_list ON grant_obj TO [user_identity] | [ROLE role_name] GRANT privilege_list ON RESOURCE grant_obj TO [user_identity] | [ROLE role_name] """ sql = "GRANT {privilege_list} ON {grant_obj} TO {role_desc}{user}" if isinstance(privilege_list, list): privilege_list = ', '.join(privilege_list) if is_role: role_desc = "ROLE " user = '"%s"' % user else: role_desc = '' user = '"%s"@"%s"' % (user, identity) if resource: grant_obj = 'RESOURCE "%s"' % resource else: grant_obj = "%s.%s.%s" % (catalog, database, table) grant_obj = grant_obj.strip(".") result = self.execute(sql.format(privilege_list=privilege_list, grant_obj=grant_obj, role_desc=role_desc, user=user)) if result != (): LOG.error(L("GRANT fail", user=user, privilege=privilege_list, database_name=database)) return False return True def revoke(self, user, privilege_list, database='', table='', catalog='', resource=None, is_role=False): """ revoke REVOKE privilege_list ON db_name[.tbl_name] FROM [user_identity] | [ROLE role_name] REVOKE privilege_list ON RESOURCE resource_name FROM [user_identity] | [ROLE role_name] """ if isinstance(privilege_list, list): privilege_list = ', '.join(privilege_list) sql = "REVOKE {privilege_list} ON {revoke_obj} FROM {role_desc}{user}" if is_role: role_desc = 'ROLE ' user = '"%s"' % user else: role_desc = '' if resource is None: revoke_obj = "%s.%s.%s" % (catalog, database, table) revoke_obj = revoke_obj.strip('.') else: revoke_obj = 'RESOURCE %s' % resource result = self.execute(sql.format(privilege_list=privilege_list, revoke_obj=revoke_obj, role_desc=role_desc, user=user)) if result != (): LOG.error(L("REVOKE fail", sql=sql, msg=str(result))) return False return True def get_grant(self, user=None, all=None): """show grant""" if user is None: sql = 'SHOW GRANTS' else: sql = 'SHOW GRANTS FOR `%s`' % user if all: sql = 'SHOW ALL GRANTS' result = self.__execute_and_rebuild_meta_class(sql, palo_job.GrantInfo) return result def is_master(self): """ is master deprecated """ sql = "SHOW FRONTENDS" fe_list = self.execute(sql) for fe in fe_list: if fe[palo_job.FrontendInfo.Host] == socket.gethostbyname(self.host): return fe[palo_job.FrontendInfo.IsMaster] == "true" return False def get_alive_backend_list(self): """ 获取alive backend """ backend_list = self.get_backend_list() result = list() for backend in backend_list: if palo_job.BackendProcInfo(backend).get_alive() == "true": result.append(backend) LOG.info(L("GET ALIVE BACKEND", alive_be=result)) return result def get_backend(self, backend_id): """ 获取backend """ backend_list = self.get_backend_list() for backend in backend_list: if palo_job.BackendProcInfo(backend).get_backend_id() == str(backend_id): return backend LOG.warning(L("Get no backend by backend id", backend_id=backend_id)) return None def get_backend_heartbeat_port(self, value=None, idx=None): """get be hearbeat port""" be_list = self.get_backend_list() if value is None: port = palo_job.BackendProcInfo(be_list[0]).get_heartbeatport() else: port = util.get_attr_condition_value(be_list, idx, value, palo_job.BackendProcInfo.HeartbeatPort) return port def get_backend_list(self): """ 获取backend """ sql = "SHOW BACKENDS" result = self.__execute_and_rebuild_meta_class(sql, palo_job.BackendProcInfo) return result def get_backend_id_list(self): """get backend id list""" backend_list = self.get_backend_list() id_list = list() for be in backend_list: id_list.append(be[0]) return id_list def get_be_hostname_by_id(self, be_id): """ 获取backend hostname by id """ be = self.get_backend(be_id) if not be: return None return palo_job.BackendProcInfo(be).get_hostname() def get_backend_host_list(self): """ 返回活动状态的backend的ip """ backend_list = self.get_alive_backend_list() be_host_list = [palo_job.BackendProcInfo(backend).get_ip() for backend in backend_list] return tuple(be_host_list) def get_backend_host_ip(self): """get all be host ip""" res = [] backend_list = self.get_backend_list() for backend in backend_list: cur_ip = palo_job.BackendProcInfo(backend).get_ip() res.append(cur_ip) return res def get_backend_host_name(self): """get all be host name""" backend_list = self.get_backend_list() return util.get_attr(backend_list, palo_job.BackendProcInfo.Host) def get_backend_host_port_list(self): """ 返回活动状态的backend的host:port metadata changed backends[3] is be hostname, backends[4] is be heartbeat port """ backend_list = self.get_alive_backend_list() backend_host_port_list = [] for backend in backend_list: backend_info = palo_job.BackendProcInfo(backend) backend_host_port = '%s:%s' % (backend_info.get_hostname(), backend_info.get_heartbeatport()) backend_host_port_list.append(backend_host_port) return tuple(backend_host_port_list) def add_backend_list(self, backend_list): """ 增加backend disable """ if not isinstance(backend_list, list): backend_list = [backend_list, ] for backend in backend_list: sql = 'ALTER SYSTEM ADD BACKEND "%s"' % (backend) result = self.execute(sql) if result != (): LOG.info(L('ADD BACKEND FAIL', backend=backend)) return False return True def add_backend(self, host_name, port, tag_location=None): """ 增加backend """ sql = 'ALTER SYSTEM ADD BACKEND "%s:%s"' % (host_name, port) if tag_location is not None: sql = '%s PROPERTIES("tag.location"="%s")' % (sql, tag_location) result = self.execute(sql) time.sleep(2) if result != (): LOG.info(L('ADD BACKEND FAIL', backend=host_name, port=port, tag_location=tag_location)) return False LOG.info(L('ADD BACKEND SUCCESS', backend=host_name, port=port, tag_location=tag_location)) return True def drop_backend_list(self, backend_list): """ 移除backend """ if not isinstance(backend_list, list): backend_list = [backend_list, ] for backend in backend_list: sql = 'ALTER SYSTEM DROPP BACKEND "%s"' % (backend) result = self.execute(sql) if result != (): LOG.info(L('DROP BACKEND FAIL', backend=backend)) return False time.sleep(2) return True def decommission_backend_list(self, backend_list): """be下线""" if not isinstance(backend_list, list): backend_list = [backend_list, ] for backend in backend_list: sql = 'ALTER SYSTEM DECOMMISSION BACKEND "%s"' % (backend) result = self.execute(sql) if result != (): LOG.info(L('DECOMMISSION BACKEND FAIL', backend=backend)) return False return True def add_fe_list(self, fe_list, type='OBSERVER'): """ 增加FE type: OBSERVER, REPLICA """ if not isinstance(fe_list, list): fe_list = [fe_list, ] for fe in fe_list: sql = 'ALTER SYSTEM ADD %s "%s"' % (type, fe) result = self.execute(sql) if result != (): LOG.info(L('ADD FE FAILED', fe=fe)) return False return True def drop_fe_list(self, fe_list, type='OBSERVER'): """ 增加FE type: OBSERVER, REPLICA """ if not isinstance(fe_list, list): fe_list = [fe_list, ] for fe in fe_list: sql = 'ALTER SYSTEM DROP %s "%s"' % (type, fe) result = self.execute(sql) if result != (): LOG.info(L('DROP FE FAILED', fe=fe)) return False return True def get_fe_list(self): """ 获取FE list """ sql = "SHOW FRONTENDS" result = self.execute(sql) return result def get_fe_host_port_list(self, type=None): """ 返回fe的host:port type: OBSERVER, REPLICA """ fe_list = self.get_fe_list() fe_host_port_list = [] for fe in fe_list: if type is not None and fe[2] != type: continue fe_host_port = '%s:%s' % (fe[0], fe[1]) fe_host_port_list.append(fe_host_port) return tuple(fe_host_port_list) def get_master(self): """ 返回master的host:port """ fe_list = self.get_fe_list() for fe in fe_list: fe_info = palo_job.FrontendInfo(fe) if fe_info.get_ismaster() == "true": fe_host_port = "%s:%s" % (fe_info.get_host(), fe_info.get_httpport()) return fe_host_port return None def get_master_host(self): """ 返回master的host:port """ fe_list = self.get_fe_list() for fe in fe_list: fe_info = palo_job.FrontendInfo(fe) if fe_info.get_ismaster() == "true": return fe_info.get_host() return None def get_fe_LastHeartbeat(self, fe_ip): """ 返回fe_LastHeartbeat """ fe_list = self.get_fe_list() for fe in fe_list: if palo_job.FrontendInfo(fe).get_IP() == str(fe_ip): return fe[palo_job.FrontendInfo(fe).get_LastHeartbeat()] LOG.warning(L("Get no fe by fe id", fe_id=fe_ip)) return None def get_fe_host(self): """get fe host list""" fe_list = self.get_fe_list() fe_host = list() for fe in fe_list: fe_host.append(fe[1]) return fe_host def recover_database(self, database_name): """ 恢复database """ sql = "RECOVER DATABASE %s" % database_name try: self.execute(sql) except PaloException as e: LOG.error(L("recover database error", fe=str(self), database_name=database_name, \ error=e)) return False return True def recover_table(self, table_name, database_name=None): """ 恢复database """ database_name = self.database_name if database_name is None else database_name sql = "RECOVER TABLE " if database_name is not None: sql = '%s%s.' % (sql, database_name) sql = '%s%s' % (sql, table_name) try: self.execute(sql) except PaloException as e: LOG.error(L("recover table error", fe=str(self), database_name=database_name, \ table_name=table_name, error=e)) return False return True def recover_partition(self, table_name, partition_name, database_name=None): """ 恢复database """ database_name = self.database_name if database_name is None else database_name sql = "RECOVER PARTITION %s FROM " % (partition_name) if database_name is not None: sql = '%s%s.' % (sql, database_name) sql = '%s%s' % (sql, table_name) try: self.execute(sql) except PaloException as e: LOG.error(L("recover partition error", fe=str(self), database_name=database_name, \ table_name=table_name, partition_name=partition_name, error=e)) return False return True def rename_database(self, new_database_name, old_database_name=None): """ 重命名数据库 """ if old_database_name is None: old_database_name = self.database_name sql = 'ALTER DATABASE %s RENAME %s' % (old_database_name, new_database_name) try: self.execute(sql) except PaloException as e: LOG.error(L("rename database error", fe=str(self), new_database_name=new_database_name, error=e)) return False return True def rename_table(self, new_table_name, old_table_name, database_name=None): """ 重命名数据库 """ database_name = self.database_name if database_name is None else database_name sql = 'ALTER TABLE %s.%s RENAME %s' % (database_name, old_table_name, new_table_name) try: self.execute(sql) except PaloException as e: LOG.error(L("rename table error", fe=str(self), error=e)) return False return True def rename_rollup(self, new_index_name, old_index_name, table_name, database_name=None): """ 重命名数据库 """ database_name = self.database_name if database_name is None else database_name sql = 'ALTER TABLE %s.%s RENAME ROLLUP %s %s' % (database_name, table_name, old_index_name, new_index_name) try: self.execute(sql) except PaloException as e: LOG.error(L("rename rollup error", fe=str(self), error=e)) return False return True def rename_partition(self, new_partition_name, old_partition_name, table_name, database_name=None): """ 重命名数据库 """ database_name = self.database_name if database_name is None else database_name sql = 'ALTER TABLE %s.%s RENAME PARTITION %s %s' % (database_name, table_name, old_partition_name, new_partition_name) try: self.execute(sql) except PaloException as e: LOG.error(L("rename partition error", fe=str(self), error=e)) return False return True def show_databases(self, database_name=None): """show databases """ sql = 'SHOW DATABASES' if database_name: sql = 'SHOW DATABASES LIKE "%s"' % database_name return self.execute(sql) def show_tables(self, table_name=None): """show tables """ sql = 'SHOW TABLES' if table_name: sql = 'SHOW TABLES LIKE "%s"' % table_name return self.execute(sql) def show_partitions(self, table_name, database_name=None): """show partitions """ database_name = self.database_name if database_name is None else database_name sql = 'SHOW PARTITIONS FROM %s.%s' % (database_name, table_name) result = self.__execute_and_rebuild_meta_class(sql, palo_job.PartitionInfo) return self.execute(sql) def show_loading_job_state(self, database_name=None, state='LOADING'): """show loading state job""" ret = self.show_load(database_name=database_name, state=state) return ret def show_load(self, database_name=None, label=None, state=None, order_by=None, limit=None, offset=None): """ SHOW LOAD [FROM db_name] [ WHERE [LABEL [ = "your_label" | LIKE "label_matcher"]] [STATE = ["PENDING"|"ETL"|"LOADING"|"FINISHED"|"CANCELLED"|]] ] [ORDER BY ...] [LIMIT limit][OFFSET offset]; Returns: load_job """ sql = 'SHOW LOAD' if database_name: sql = '%s FROM %s' % (sql, database_name) if label: sql = '%s WHERE label = "%s"' % (sql, label) if state: sql = '%s WHERE STATE="%s"' % (sql, state) if order_by: sql = '%s ORDER BY %s' % (sql, order_by) if limit: sql = '%s LIMIT %s' % (sql, limit) if offset: sql = '%s OFFSET %s' % (sql, offset) return self.execute(sql) def desc_table(self, table_name, database_name=None, is_all=False): """desc table""" database_name = self.database_name if database_name is None else database_name if is_all: sql = 'DESC %s.%s all' % (database_name, table_name) else: sql = 'DESC %s.%s' % (database_name, table_name) return self.execute(sql) def show_schema_change_job(self, database_name=None, table_name=None, state=None): """show schema change job""" database_name = self.database_name if database_name is None else database_name where_list = [] if table_name: where_list.append('TableName = "%s"' % table_name) if state: where_list.append('State= "%s"' % state) if len(where_list) == 0: sql = 'SHOW ALTER TABLE COLUMN FROM %s' % database_name return self.execute(sql) else: sql = 'SHOW ALTER TABLE COLUMN FROM %s WHERE %s' % (database_name, ' AND '.join(where_list)) return self.execute(sql) def show_rollup_job(self, database_name=None, table_name=None): """show schema change job""" database_name = self.database_name if database_name is None else database_name sql = 'SHOW ALTER TABLE ROLLUP FROM %s' % database_name if table_name is None: return self.execute(sql) else: sql = '%s WHERE TableName = "%s"' % (sql, table_name) sc_job = self.execute(sql) return sc_job def set_properties(self, kv_list, user=None): """set properties """ for_user = '' if user is not None: for_user = 'FOR "%s" ' % user properties = ', '.join(kv_list) if not isinstance(kv_list, str) else kv_list sql = 'SET PROPERTY %s%s' % (for_user, properties) result = self.execute(sql) if result != (): return False return True def set_max_user_connections(self, max_connections, user=None): """set max user connections """ kv = '"max_user_connections" = "%s"' % max_connections return self.set_properties(kv, user) def set_resource_cpu_share(self, cpu_share, user=None): """set resource cpu share """ kv = '"resource.cpu_share" = "%s"' % cpu_share return self.set_properties(kv, user) def set_quota_low(self, quota_low, user=None): """set quota low """ kv = '"quota.low" = "%s"' % quota_low return self.set_properties(kv, user) def set_quota_normal(self, quota_normal, user=None): """set quota normal """ kv = '"quota.normal" = "%s"' % quota_normal return self.set_properties(kv, user) def set_quota_high(self, quota_high, user=None): """set quota high """ kv = '"quota.high" = "%s"' % quota_high return self.set_properties(kv, user) def set_load_cluster_hadoop_palo_path(self, cluster_name, hadoop_palo_path, user=None): """set load cluster hadoop palo path """ kv = '"load_cluster.%s.hadoop_palo_path" = "%s"' % (cluster_name, hadoop_palo_path) return self.set_properties(kv, user) def set_load_cluster_hadoop_http_port(self, cluster_name, hadoop_http_port, user=None): """set load cluster hadoop http port """ kv = '"load_cluster.%s.hadoop_http_port" = "%s"' % (cluster_name, hadoop_http_port) return self.set_properties(kv, user) def set_load_cluster_hadoop_configs(self, cluster_name, hadoop_configs, user=None): """set load cluster hadoop configs """ kv = '"load_cluster.%s.hadoop_configs" = "%s"' % (cluster_name, hadoop_configs) return self.set_properties(kv, user) def set_load_cluster(self, cluster_name, hadoop_configs, hadoop_palo_path, hadoop_http_port=None, user=None): """set load cluster """ kv_1 = '"load_cluster.%s.hadoop_configs" = "%s"' % (cluster_name, hadoop_configs) kv_2 = '"load_cluster.%s.hadoop_palo_path" = "%s"' % (cluster_name, hadoop_palo_path) kv_list = [kv_1, kv_2] if hadoop_http_port is not None: kv_3 = '"load_cluster.%s.hadoop_http_port" = "%s"' % (cluster_name, hadoop_http_port) kv_list.append(kv_3) return self.set_properties(kv_list, user) def set_default_load_cluster(self, cluster_name, user=None): """set default load cluster """ kv = '"default_load_cluster" = "%s"' % cluster_name return self.set_properties(kv, user) def remove_default_load_cluster(self, user=None): """remove default load cluster """ kv = '"default_load_cluster" = ""' return self.set_properties(kv, user) def remove_load_cluster(self, cluster_name, user=None): """remove load cluster hadoop configs """ kv = '"load_cluster.%s" = ""' % cluster_name return self.set_properties(kv, user) def remove_load_cluster_hadoop_configs(self, cluster_name, user=None): """remove load cluster hadoop configs """ kv = '"load_cluster.%s.hadoop_configs" = ""' % cluster_name return self.set_properties(kv, user) def remove_load_cluster_hadoop_http_port(self, cluster_name, user=None): """remove load cluster hadoop http port """ kv = '"load_cluster.%s.hadoop_http_port" = ""' % cluster_name return self.set_properties(kv, user) def remove_load_cluster_hadoop_palo_path(self, cluster_name, user=None): """remove load cluster hadoop palo path """ kv = '"load_cluster.%s.hadoop_palo_path" = ""' % cluster_name return self.set_properties(kv, user) def gen_hadoop_configs(self, fs_default_name=None, mapred_job_tracker=None, hadoop_job_ugi=None, mapred_job_priority=None): """gen hadoop configs """ configs = '' if fs_default_name is not None: configs += 'fs.default.name=%s;' % (fs_default_name) if mapred_job_tracker is not None: configs += 'mapred.job.tracker=%s;' % (mapred_job_tracker) if hadoop_job_ugi is not None: configs += 'hadoop.job.ugi=%s;' % (hadoop_job_ugi) if mapred_job_priority is not None: configs += 'mapred.job.priority=%s;' % (mapred_job_priority) configs = configs.rstrip(';') return configs def show_property(self, key=None, user=None): """show property """ for_user = '' if user is None else ' FOR "%s"' % user property_key = '' if key is None else ' LIKE "%s" ' % key sql = 'SHOW PROPERTY%s%s' % (for_user, property_key) return self.execute(sql) def show_max_user_connections(self, user=None): """show max user connections ((u'max_user_connections', u'10'),) """ result = self.show_property('max_user_connections', user) return int(result[0][1]) def show_resource_cpu_share(self, user=None): """show resource cpu share ((u'resource.cpu_share', u'1000'), (u'resource.hdd_read_iops', u'80'), (u'resource.hdd_read_mbps', u'30'), (u'resource.io_share', u'1000'), (u'resource.ssd_read_iops', u'1000'), (u'resource.ssd_read_mbps', u'30')) """ result = self.show_property('resource', user) for item in result: if item[0] == 'resource.cpu_share': return int(item[1]) def show_quota_low(self, user=None): """show quota low """ result = self.show_property('quota', user) for item in result: if item[0] == 'quota.low': return int(item[1]) def show_quota_normal(self, user=None): """show quota normal """ result = self.show_property('quota', user) for item in result: if item[0] == 'quota.normal': return int(item[1]) def show_quota_high(self, user=None): """show quota high """ result = self.show_property('quota', user) for item in result: if item[0] == 'quota.high': return int(item[1]) def show_load_cluster(self, user=None): """show load cluster """ result = self.show_property('%load_cluster%', user) load_cluster_dict = {} for k, v in result: load_cluster_dict[k] = v return load_cluster_dict def add_whitelist(self, user, whitelist): """ add whitelist """ sql = 'ALTER USER %s ADD WHITELIST "%s"' % (user, whitelist) return self.execute(sql) == () def delete_whitelist(self, user, whitelist): """ delete whitelist """ sql = 'ALTER USER %s DELETE WHITELIST "%s"' % (user, whitelist) return self.execute(sql) == () def show_whitelist(self, user): """ show whitelist """ sql = 'SHOW WHITELIST' result = self.execute(sql) for r in result: if r[0] == user: return r[1] return None def clean_whitelist(self, user): """ clean whitelist """ whitelist = self.show_whitelist(user) if whitelist is None: return for w in whitelist.split(','): if w: self.delete_whitelist(user, w) def create_cluster(self, cluster_name, instance_num, password=''): """ create cluster """ sql = 'CREATE CLUSTER %s PROPERTIES ( "instance_num" = "%d") ' \ 'IDENTIFIED BY "%s"' % (cluster_name, instance_num, password) self.execute(sql) def enter(self, cluster_name): """ create cluster """ sql = 'ENTER %s' % (cluster_name) self.execute(sql) self.cluster_name = cluster_name def clean_cluster(self, cluster_name=None): """ clean cluster """ if not cluster_name: cluster_name_list = self.get_cluster_list() for cluster_name in cluster_name_list: if cluster_name == 'default_cluster': continue self.enter(cluster_name) self.clean() self.drop_cluster(cluster_name) else: try: self.enter(cluster_name) self.clean() self.drop_cluster(cluster_name) except: pass def drop_cluster(self, cluster_name): """ drop cluster """ sql = 'DROP CLUSTER %s' % (cluster_name) self.execute(sql) def get_cluster_list(self): """ show cluster """ sql = 'SHOW CLUSTERS' records = self.execute(sql) return tuple([r[0] for r in records]) def alter_cluster(self, cluster_name, instance_num): """ alter cluster """ sql = 'ALTER CLUSTER %s PROPERTIES ( "instance_num" = "%d" )' \ % (cluster_name, instance_num) self.execute(sql) def link(self, src_cluster_name, src_database_name, dst_cluster_name, dst_database_name): """ link """ sql = 'LINK DATABASE %s.%s %s.%s' % (src_cluster_name, src_database_name, dst_cluster_name, dst_database_name) self.execute(sql) def migrate(self, src_cluster_name, src_database_name, dst_cluster_name, dst_database_name): """ migrate """ sql = 'MIGRATE DATABASE %s.%s %s.%s' % (src_cluster_name, src_database_name, dst_cluster_name, dst_database_name) self.execute(sql) def get_migrate_status(self, src_cluster_name, src_database_name, dst_cluster_name, dst_database_name): """ 获取指定状态的导入任务信息 """ sql = 'SHOW MIGRATIONS' migrate_list = self.execute(sql) print(migrate_list) for it in migrate_list: if it[1] == '%s:%s' % (src_cluster_name, src_database_name) \ and it[2] == '%s:%s' % (dst_cluster_name, dst_database_name): return it[3] return None def wait_migrate(self, src_cluster_name, src_database_name, dst_cluster_name, dst_database_name): """ wait migrate """ time.sleep(5) ret = self.get_migrate_status(src_cluster_name, src_database_name, dst_cluster_name, dst_database_name) if not ret: return False while True: ret = self.get_migrate_status(src_cluster_name, src_database_name, dst_cluster_name, dst_database_name) if ret == '100%': return True time.sleep(1) def add_broker(self, broker_name, host_port): """ add broker """ sql = 'ALTER SYSTEM ADD BROKER %s "%s"' % (broker_name, host_port) return self.execute(sql) == () def drop_broker(self, broker_name, host_port): """ add broker """ sql = 'ALTER SYSTEM DROP BROKER %s "%s"' % (broker_name, host_port) return self.execute(sql) == () def get_broker_list(self): """ show whitelist """ sql = 'SHOW PROC "/brokers"' result = self.execute(sql) return result def get_broker_start_update_time(self): """show broker_start_update_time""" res = self.get_broker_list() if res: LOG.info(L("get_broker_start_update_time", start_time=palo_job.BrokerInfo(res[0]).get_last_start_time(), update_time=palo_job.BrokerInfo(res[0]).get_last_update_time())) return palo_job.BrokerInfo(res[0]).get_last_start_time() return None def drop_all_broker(self, broker_name): """ add broker """ sql = 'ALTER SYSTEM DROP ALL BROKER %s"' % broker_name return self.execute(sql) == () def export(self, table_name, to_path, broker_info=None, partition_name_list=None, property_dict=None, database_name=None, where=None): """export data. Args: table_name: table name, str to_path: export path, str broker_info: broker info, BrokerInfo partition_name_list: name list of patitions, str list property_dict: properties, dict str->str database_name: database name, str where: str, k1>0 Returns: True if succeed """ sql = 'EXPORT TABLE' if not database_name: database_name = self.database_name sql = '%s %s.%s' % (sql, database_name, table_name) if partition_name_list: sql = '%s PARTITION (%s)' % (sql, ','.join(partition_name_list)) if where: sql = '%s WHERE %s' % (sql, where) sql = '%s TO "%s"' % (sql, to_path) if property_dict: p = '' for d in property_dict: p = '%s "%s"="%s",' % (p, d, property_dict[d]) p = p.rstrip(",") sql = '%s PROPERTIES(%s)' % (sql, p) if broker_info: sql = '%s %s' % (sql, str(broker_info)) return self.execute(sql) == () def get_export_status(self, database_name=None): """get export status. Args: database_name: database name, str Returns: True if succeed """ sql = 'SHOW EXPORT' if database_name: sql = '%s FROM %s' % (sql, database_name) sql = '%s %s' % (sql, 'ORDER BY JOBID LIMIT 1') return self.execute(sql) def show_export(self, database_name=None, state=None, export_job_id=None, order_by=None, limit=None): """ Args: database_name: database name state: export status filter, PENDING|EXPORTIING|FINISHED|CANCELLED export_job_id: export job id filter order_by: column, str 'starttime' limit: num, int Returns: export job """ sql = 'SHOW EXPORT' if database_name: sql = '%s FROM %s' % (sql, database_name) if state: sql = '%s WHERE STATE="%s"' % (sql, state) if export_job_id: sql = '%s AND ID="%s"' % (sql, export_job_id) elif export_job_id: sql = '%s WHERE ID="%s"' % (sql, export_job_id) if order_by: sql = '%s ORDER BY %s' % (sql, order_by) if limit: sql = '%s LIMIT %s' % (sql, limit) return self.execute(sql) def wait_export(self, database_name=None): """wait export. Args: database_name: database name, str Returns: True if succeed """ timeout = 1200 while timeout > 0: r = self.get_export_status(database_name) export_job = palo_job.ExportJob(r[0]) if export_job.get_state() == 'FINISHED': LOG.info(L("export succ.", state='FINISHED', database_name=database_name)) return True elif export_job.get_state() == 'CANCELLED': LOG.warning(L("export fail.", state='CANCELLED', database_name=database_name, msg=export_job.get_error_msg())) return False else: time.sleep(1) timeout -= 1 LOG.warning(L("export timeout.", timeout=timeout, database_name=database_name)) return False def create_external_table(self, table_name, column_list, engine, property, broker_property=None, database_name=None, set_null=False): """Create broker table. Args: table_name: table name, str column_list: list of columns engine: str, it should be olap, mysql, elasticsearch, broker property: map, if broker engine e.g.: PROPERTIES ( "broker_name" = "broker_name", "paths" = "file_path1[,file_path2]", "column_separator" = "value_separator" "line_delimiter" = "value_delimiter" |"format" = "parquet" ) mysql engine e.g.: PROPERTIES ( "host" = "mysql_server_host", "port" = "mysql_server_port", "user" = "your_user_name", "password" = "your_password", "database" = "database_name", "table" = "table_name" ) es engine e.g.: PROPERTIES ( "hosts" = "http://", "user" = "root", "password" = "", "index" = "new_data", "type" = "mbtable" ) broker_property: broker property, broker maybe hdfs, bos, afs. set_null: if column can be null Returns: True if succeed """ database_name = self.database_name if database_name is None else database_name # table name sql = 'CREATE EXTERNAL TABLE %s.%s (' % (database_name, table_name) # columns for column in column_list: sql = '%s %s,' % (sql, util.column_to_no_agg_sql(column, set_null)) sql = '%s ) ENGINE=%s' % (sql.rstrip(','), engine) # property sql = '%s PROPERTIES %s' % (sql, util.convert_dict2property(property)) # broker_property if isinstance(broker_property, BrokerInfo): sql = '%s BROKER PROPERTIES %s' % (sql, broker_property.get_property()) elif isinstance(broker_property, str): sql = '%s BROKER PROPERTIES (%s)' % (sql, broker_property) elif isinstance(broker_property, dict): sql = '%s BROKER PROPERTIES %s' % (sql, util.convert_dict2property(broker_property)) else: pass ret = self.execute(sql) if ret != (): LOG.info(L('CREATE EXTERNAL TABLE fail.', database_name=database_name, table_name=table_name)) return False LOG.info(L('CREATE EXTERNAL TABLE succ.', database_name=database_name, table_name=table_name)) return True def stream_load(self, table_name, data_file, database_name=None, host=None, port=None, user=None, password=None, cluster_name=None, is_wait=True, max_filter_ratio=None, load_label=None, column_name_list=None, timeout=300, column_separator=None, partition_list=None, where_filter=None, time_zone=None, **kwargs): """ Args: table_name: 必填,表名 data_file: 必填,上传数据 user: 可选,palo的用户 password: 可选, palo用户的密码 database_name: 可选, 数据库名,默认为当前client的数据库 cluster_name: 可选, cluster,默认为default_cluster.不建议使用 host: 可选，默认为当前client连接的fe port: 可选，默认为当前client的http_port -H参数包括： max_filter_ratio: 可选, 最大容忍可过滤的数据比例0-1 load_label/label: 可选,导入的标签 column_name_list/columns: 可选, 指定导入文件中的列和table中的列的对应关系 timeout: 可选,连接的超时时间 column_separator: 可选, 列的分割符 partition_list/partitions: 可选, 指定本次导入的分区 where_filter/where: 可选,用户抽取部分数据 time_zone/timezone: 可选,用户设置时区 strict_mode: 用户指定此次导入是否开启严格模式，默认为false exec_mem_limit: 导入内存限制。默认为 2GB。单位为字节 format: 指定导入数据格式，默认是csv，支持json格式 jsonpaths: 导入json方式分为：简单模式和精准模式 strip_outer_array: 为true表示json数据以数组对象开始且将数组对象中进行展平，默认值是false json_root: json_root为合法的jsonpath字符串，用于指定json document的根节点，默认值为"" 控制参数： is_wait: 当返回结果为publish timeout时，是否等待事务visible Returns: 0 fail/label already exist 1 success 2 publish timeout """ database_name = database_name if database_name is not None else self.database_name host = host if host is not None else self.host port = port if port is not None else self.http_port user = user if user is not None else self.user password = password if password is not None else self.password uri = "http://%s:%s/api/%s/%s/_stream_load" % (host, port, database_name, table_name) buf = BytesIO() c = pycurl.Curl() c.setopt(c.URL, uri) c.setopt(c.WRITEFUNCTION, buf.write) head = dict() head['Content-length'] = os.path.getsize(data_file) head['Transfer-Encoding'] = '' head['Expect'] = '100-continue' file = open(data_file, "rb") if load_label: head['label'] = load_label if column_name_list: head['columns'] = ','.join(column_name_list) if column_separator: head['column_separator'] = column_separator if max_filter_ratio: head['max_filter_ratio'] = max_filter_ratio if partition_list: head['partitions'] = ','.join(partition_list) if where_filter: head['where'] = where_filter if time_zone: head['timezone'] = time_zone if timeout: head['timeout'] = timeout print(data_file) head.update(kwargs) print(head) param = '' for k, v in head.items(): param = '%s -H "%s:%s"' % (param, k, v) curl_cmd = 'curl --location-trusted -u %s:%s %s -T %s %s' % (user, password, param, data_file, uri) LOG.info(L('STREAM LOAD CURL CMD.', cmd=curl_cmd)) print(curl_cmd) # 设置-H参数 c.setopt(pycurl.HTTPHEADER, [k + ': ' + str(v) for k, v in head.items()]) # basic认证 c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) c.setopt(pycurl.USERNAME, user) c.setopt(pycurl.PASSWORD, password) # 上传文件 c.setopt(pycurl.PUT, 1) c.setopt(pycurl.UPLOAD, 1) c.setopt(pycurl.READDATA, file) # 重定向，--location-trusted c.setopt(pycurl.UNRESTRICTED_AUTH, 1) c.setopt(pycurl.FOLLOWLOCATION, True) # 连接超时 c.setopt(pycurl.TIMEOUT, 30) LOG.info(L('STREAM LOAD.', head=head, url=uri, data_file=data_file)) try: c.perform() msg = buf.getvalue() print(msg) LOG.info(L('STREAM LOAD ret.', ret=msg)) if c.getinfo(c.HTTP_CODE) == 200: ret = json.loads(msg) status = ret.get("Status") txn_id = ret.get("TxnId") if status == 'Success': stream_load_ret = 1 self.wait_txn(txn_id, database_name) elif status == 'Publish Timeout': stream_load_ret = 2 if is_wait: self.wait_txn(txn_id, database_name) else: # Label Already Exists/Fail url = ret.get("ErrorURL") if url: sql = 'show load warnings on "%s"' % url ret = self.execute(sql) print(ret[0]) stream_load_ret = 0 else: stream_load_ret = 0 except Exception as e: stream_load_ret = 0 print(str(e)) LOG.info(L('STREAM LOAD FAILED.', msg=str(e))) finally: buf.close() file.close() c.close() return stream_load_ret def insert_select(self, table_name, select_query, column_name_list=None, is_streaming=True, database_name=None): """insert""" if database_name is not None: sql = 'INSERT INTO %s.%s' % (database_name, table_name) else: sql = 'INSERT INTO %s' % table_name if column_name_list: sql = '%s (%s)' % (sql, ','.join(column_name_list)) sql = '%s %s' % (sql, select_query) ret = self.execute(sql) return ret == () def get_column(self, column_name, table_name, database_name=None): """get column""" column_tuple = self.desc_table(table_name, database_name) for column in column_tuple: if column_name == palo_job.DescInfo(column).get_field(): return column_name return None def get_column_info(self, column_name, table_name, database_name=None): """get column info""" column_tuple = self.desc_table(table_name, database_name) for column in column_tuple: if column_name == palo_job.DescInfo(column).get_field(): return column return None def get_all_columns(self, table_name, database_name=None): """get table columns""" column_tuple = self.desc_table(table_name, database_name) column_name_list = list() for column in column_tuple: column_name = palo_job.DescInfo(column).get_field() column_name_list.append(column_name) return column_name_list def create_repository(self, repo_name, broker_name, repo_location, repo_properties, is_read_only=False): """ Args: repo_name: string, repository name broker_name: string, broker name repo_location: string, repository location path repo_properties: dict, property of repository location "bos_endpoint" = "http://gz.bcebos.com", "bos_accesskey" = "XXXXXXXXXXXXXXXXXXXX", "bos_secret_accesskey"="XXXXXXXXXXXXXXXXX" properties = dict() properties["bos_endpoint"] = "http://gz.bcebos.com" properties["bos_accesskey"] = "XXXXXXXXXXXXXXXXXXX" properties["bos_secret_accesskey"] = "XXXXXXXXXXXXXXXXXXX" Returns: True create repo success False create repo failed need to check """ sql = 'CREATE {readonly} REPOSITORY {repo_name} WITH BROKER {broker_name} \ ON LOCATION "{repo_location}" PROPERTIES ({repo_properties})' if isinstance(repo_properties, dict): property = '' for k, v in repo_properties.items(): property += ' "%s" = "%s",' % (k, v) p = property.strip(',') elif isinstance(repo_properties, str): p = repo_properties else: return False if is_read_only: readonly = 'READ ONLY' else: readonly = '' s = sql.format(readonly=readonly, repo_name=repo_name, broker_name=broker_name, repo_location=repo_location, repo_properties=p) try: ret = self.execute(s) except Exception as e: logging.exception(e) LOG.info(L('CREATE REPO fail.', repository_name=repo_name, msg=str(e))) return False if ret == (): LOG.info(L('CREATE REPO succ.', repository_name=repo_name)) return True else: LOG.info(L('CREATE REPO fail.', repository_name=repo_name)) return False def drop_repository(self, repo_name): """ Args: repo_name: string, repository name to be dropped Returns: True drop repo success False drop repo failure """ sql = 'DROP REPOSITORY {repo_name}' s = sql.format(repo_name=repo_name) try: ret = self.execute(s) except Exception as e: logging.exception(e) LOG.info(L('DROP REPO fail.', repository_name=repo_name, msg=str(e))) return False if ret == (): LOG.info(L('DROP REPO succ.', repository_name=repo_name)) return True else: LOG.info(L('DROP REPO fail.', repository_name=repo_name)) return False def show_repository(self): """ Returns: existed repo RepoId：唯一的仓库ID RepoName：仓库名称 CreateTime：第一次创建该仓库的时间 IsReadOnly：是否为只读仓库 Location：仓库中用于备份数据的根目录 Broker：依赖的 Broker ErrMsg： Palo 会定期检查仓库的连通性，如果出现问题，这里会显示错误信息 """ sql = 'SHOW REPOSITORIES' ret = self.execute(sql) return ret def get_repository(self, repo_name=None, repo_info=False): """ Args: repo_name: string, repo name Returns: if repo_name exists return (RepoId, RepoName, CreateTime, IsReadOnly, Location, Broker, ErrMsg) else return None """ r = self.show_repository() if repo_name is None: return util.get_attr(r, palo_job.RepoInfo.RepoName) for repo in r: if repo_name == repo[palo_job.RepoInfo.RepoName]: if not repo_info: return repo else: return palo_job.RepoInfo(repo) return None def backup(self, snapshot_label, backup_list, repo_name, database_name=None, type=None, timeout=1800, is_wait=False): """ Args: snapshot_label: string, snapshot name backup_list: list, table/partition to backup, ['table', 'table2 PARTITION (p1, p2)'] repo_name: string, repo name database_name: string, db name, if none, use default db type: string, only support full, full backup timeout: int, timeout is_wait: False/True, if wait backup job finish Returns: True backup success False backup failure """ sql = 'BACKUP SNAPSHOT {db}.{snapshot_label} TO {repo} on ({backup}) {property}' b = ','.join(backup_list) property = 'PROPERTIES({property})' database_name = self.database_name if database_name is None else database_name if type is None and timeout is None: property = '' else: p = list() if type and type.upper() == 'FULL': p.append('"type"="full"') if timeout: p.append('"timeout"="%s"' % timeout) property = property.format(property='e,'.join(p)) s = sql.format(db=database_name, snapshot_label=snapshot_label, repo=repo_name, backup=b, property=property) try: ret = self.execute(s) except Exception as e: logging.exception(e) LOG.info(L('BACKUP SNAPSHOT FAILED.', snapshot_label=snapshot_label, msg=str(e))) msg = 'Can only run one backup or restore job of a database at same time' if msg in str(e): ret = self.show_backup(database_name) LOG.info(L("Currrent backup jobs", ret=ret)) ret = self.show_restore(database_name) LOG.info(L('Currrent restore job', ret=ret)) return False if ret != (): LOG.info(L('BACKUP SNAPSHOT FAILED.', snapshot_label=snapshot_label)) return False if is_wait is False: LOG.info(L('BACKUP SNAPSHOT SUCCEEDED.', snapshot_label=snapshot_label)) return True else: ret = self.wait_backup_job(snapshot_label) return ret def wait_backup_job(self, label=None, database_name=None): """ Args: label: if label is None, wait all backup job finish, else wait label job finish(backup job can not run at the same time in a db) Returns: True if backup job is Finished state False if backup job is Cancelled state """ ret = self.show_backup(database_name) if ret == (): return True flag = False while label is None and flag is False: flag = True for backup_job in ret: s = backup_job[palo_job.BackupJob.State] if s != 'FINISHED' and s != 'CANCELLED': flag = False time.sleep(1) ret = self.show_backup(database_name) if flag is True: LOG.info(L('BACKUP JOB ALL FINISHED, NO JOB RUNNING.')) return True timeout = 3600 while timeout > 0: for backup_job in ret: if label == backup_job[palo_job.BackupJob.SnapshotName]: s = backup_job[palo_job.BackupJob.State] if s == 'FINISHED': LOG.info(L('BACKUP SNAPSHOT FINISEHD.', snapshot_label=label)) return True elif s == 'CANCELLED': LOG.info(L('BACKUP SNAPSHOT CANCELLED.', snapshot_label=label, msg=backup_job[palo_job.BackupJob.Status])) return False time.sleep(3) ret = self.show_backup(database_name) timeout -= 1 LOG.info(L('BACKUP JOB WAIT TIMEOUT.', snapshot_label=label)) return False def show_backup(self, database_name=None): """ Args: database_name: string, database name Returns: ((), (),...) """ if database_name is None: sql = 'SHOW BACKUP' else: sql = 'SHOW BACKUP FROM {db}'.format(db=database_name) ret = self.__execute_and_rebuild_meta_class(sql, palo_job.BackupJob) return ret def cancel_backup(self, database_name=None): """ Args: database_name: string, database name Returns: True if cancel backup job success False if cancel backup job failure """ database_name = self.database_name if database_name is None else database_name sql = 'CANCEL BACKUP FROM {db}'.format(db=database_name) ret = self.execute(sql) if ret == (): LOG.info(L('CANCEL BACKUP SUCCESS.', database_name=database_name)) return True else: LOG.info(L('CANCEL BACKUP FAILURE.', database_name=database_name)) return False def restore(self, snapshot_name, repo_name, restore_list, database_name=None, replication_num=None, timeout=1800, is_wait=False): """ Args: snapshot_name: string, snapshot name repo_name: string, repo name restore_list: list, table and partition and rename, ['table1', 'table2 PARTITION (p1, p2)', 'table3 PARTITION (p1, p2) AS table_rename'] database_name: string, database name replication_num: int, replication number timeout: int, timeout is_wait: True/False, if wait restore job finished Returns: True if restore succeeded False if restore failed """ sql = 'RESTORE SNAPSHOT {db}.{snapshot_name} from {repo_name} on ({restore_list}) ' \ 'PROPERTIES("backup_timestamp"="{timestamp}"{property})' database_name = self.database_name if database_name is None else database_name r = ','.join(restore_list) p = '' if replication_num: p = '%s, "replication_num"="%d"' % (p, replication_num) if timeout: p = '%s, "timeout"="%d"' % (p, timeout) backup_timestamp = self.__get_backup_timestamp(repo_name, snapshot_name) if not backup_timestamp: LOG.info(L('get timestamp error when restore.', snapshot=snapshot_name, repo=repo_name)) return False s = sql.format(db=database_name, snapshot_name=snapshot_name, repo_name=repo_name, restore_list=r, timestamp=backup_timestamp, property=p) try: ret = self.execute(s) except Exception as e: logging.exception(e) LOG.info(L('RESTORE SNAPSHOT FAILED.', snapshot_name=snapshot_name, msg=str(e))) msg = 'Can only run one backup or restore job of a database at same time' if msg in str(e): ret = self.show_restore(database_name) LOG.info(L('Currrent restore job', ret=ret)) ret = self.show_backup(database_name) LOG.info(L('Currrent backup job', ret=ret)) return False if ret != (): LOG.info(L('RESTORE SNAPSHOT FAILED.', snapshot_name=snapshot_name)) return False if is_wait is False: LOG.info(L('RESTORE SNAPSHOT SUCCEED.', snapshot_name=snapshot_name)) return True else: r = self.wait_restore_job(database_name) return r def show_restore(self, database_name=None): """ Args: database_name: string, database name Returns: ((), (), ()) """ if database_name is None: sql = 'SHOW RESTORE' else: sql = 'SHOW RESTORE FROM %s' % database_name ret = self.__execute_and_rebuild_meta_class(sql, palo_job.RestoreJob) return ret def wait_restore_job(self, database_name=None): """wait restore job finished""" ret = self.show_restore(database_name) if ret == (): return True flag = False while flag is False: flag = True for restore_job in ret: s = restore_job[palo_job.RestoreJob.State] if s != 'FINISHED' and s != 'CANCELLED': flag = False time.sleep(3) ret = self.show_restore(database_name) LOG.info(L('RESTORE JOB FINISHED.', state=ret[-1][palo_job.RestoreJob.State], status=ret[-1][palo_job.RestoreJob.Status])) return 'FINISHED' == ret[-1][palo_job.RestoreJob.State] def cancel_restore(self, database_name=None): """ Args: database_name: string, database name Returns: True if cancel restore job succeed False if cancel resotre job fail """ database_name = self.database_name if database_name is None else database_name sql = 'CANCEL RESTORE FROM %s' % database_name ret = self.execute(sql) if ret == (): LOG.info(L('CANCEL RESTORE JOB SUCCEED.', database_name=database_name)) return True else: LOG.info(L('CANCEL RESTORE JOB FAIL.', database_name=database_name)) return False def show_snapshot(self, repo_name, snapshot=None, timestamp=None): """ Args: repo_name: string, repo name snapshot: string, snapshot name(label) timestamp: string, backup timestamp Returns: ((), ()) """ sql = 'SHOW SNAPSHOT ON %s' % repo_name if snapshot is None and timestamp is None: r = self.execute(sql) return r sql = '%s WHERE' % sql if snapshot: sql = '%s SNAPSHOT = "%s"' % (sql, snapshot) if timestamp: sql = '%s AND TIMESTAMP = "%s"' % (sql, timestamp) elif timestamp: sql = '%s TIMESTAMP = "%s"' % (sql, timestamp) r = self.execute(sql) return r def __get_backup_timestamp(self, repo_name, snapshot_name): """ Args: snapshot_name: string, snapshot name (label) Returns: timestamp: string """ r = self.show_snapshot(repo_name, snapshot=snapshot_name) if len(r) == 1: return r[0][palo_job.SnapshotInfo.Timestamp] else: return None def routine_load(self, table_name, routine_load_job_name, routine_load_property, database_name=None, data_source='KAFKA'): """ Args: table_name: string, table name routine_load_job_name: string, routine load job name routine_load_property: database_name: RoutineLoadProperty class, routine load properties: load property, job property, data source property data_source: string, data source like KAFKA Returns: if create routine load ok """ create_sql = 'CREATE ROUTINE LOAD {routine_load_job_name} ON {table_name} ' \ '{load_property} {job_property} FROM {data_source} {data_source_property}' if database_name is not None: routine_load_job_name = '%s.%s' % (database_name, routine_load_job_name) if not isinstance(routine_load_property, RoutineLoadProperty): LOG.info(L('CREATE ROUTINE LOAD ERROR. routine load property should be class RoutineLoadProperty')) sql = create_sql.format(routine_load_job_name=routine_load_job_name, table_name=table_name, load_property=routine_load_property.load_property, job_property=routine_load_property.job_property, data_source=data_source, data_source_property=routine_load_property.data_source_property) try: ret = self.execute(sql) LOG.info(L('CREATE ROUTINE LOAD JOB OK.')) return ret == () except Exception as e: LOG.info(L('CREATE ROUTINE LOAD JOB ERROR.', msg=str(e))) return False def pause_routine_load(self, routine_load_job_name, database_name=None): """pause routine load job""" if database_name is None: sql = 'PAUSE ROUTINE LOAD FOR %s' % routine_load_job_name else: sql = 'PAUSE ROUTINE LOAD FOR %s.%s' % (database_name, routine_load_job_name) try: ret = self.execute(sql) LOG.info(L('PAUSE ROUTINE LOAD OK', name=routine_load_job_name)) return ret == () except Exception as e: LOG.info(L('PAUSE ROUTINE LOAD ERROR', name=routine_load_job_name, msg=str(e))) return False def resume_routine_load(self, routine_load_job_name, database_name=None): """resume routine load""" if database_name is None: sql = 'RESUME ROUTINE LOAD FOR %s' % routine_load_job_name else: sql = 'RESUME ROUTINE LOAD FOR %s.%s' % (database_name, routine_load_job_name) try: ret = self.execute(sql) LOG.info(L('RESUME ROUTINE LOAD OK', name=routine_load_job_name)) return ret == () except Exception as e: LOG.info(L('RESUME ROUTINE LOAD ERROR', name=routine_load_job_name, msg=str(e))) return False def stop_routine_load(self, routine_load_job_name, database_name=None): """stop routine load""" if database_name is None: job_name = routine_load_job_name else: job_name = '%s.%s' % (database_name, routine_load_job_name) sql = 'STOP ROUTINE LOAD FOR %s' % job_name try: ret = self.execute(sql) LOG.info(L('STOP ROUTINE LOAD OK', name=routine_load_job_name)) show = self.execute('SHOW ALL ROUTINE LOAD FOR %s' % job_name) LOG.info(L('SHOW STOPPED ROUTINE LOAD', ret=show)) return ret == () except Exception as e: LOG.info(L('STOP ROUTINE LOAD ERROR', name=routine_load_job_name, msg=str(e))) show = self.execute('SHOW ALL ROUTINE LOAD FOR %s' % job_name) LOG.info(L('SHOW STOPPED ROUTINE LOAD', ret=show)) return False def show_routine_load(self, routine_load_job_name=None, database_name=None, is_all=False): """show routine load""" if is_all is False: all_word = '' else: all_word = 'ALL' sql = 'SHOW ROUTINE LOAD' if routine_load_job_name is None: routine_load_job_name = '' sql = 'SHOW {all} ROUTINE LOAD'.format(all=all_word) else: if database_name is not None: routine_load_job_name = '%s.%s' % (database_name, routine_load_job_name) else: routine_load_job_name = routine_load_job_name sql = 'SHOW {all} ROUTINE LOAD FOR {routine_load_job}'.format(all=all_word, routine_load_job=routine_load_job_name) try: ret = self.execute(sql) return ret except Exception as e: LOG.info(L('SHOW ROUTINE LOAD ERROR', msg=str(e))) return None def show_routine_load_task(self, routine_load_job_name): """show routine load task""" sql = 'SHOW ROUTINE LOAD TASK WHERE JOBNAME="%s"' % routine_load_job_name ret = self.execute(sql) return ret def get_routine_load_state(self, routine_load_job_name, database_name=None): """get routine load state""" ret = self.show_routine_load(routine_load_job_name, database_name=database_name) if ret == () or ret is None: ret = self.show_routine_load(routine_load_job_name, database_name=database_name, is_all=True) if ret == () or ret is None: return None LOG.info(L('GET ROUTINE LOAD STATE', state=palo_job.RoutineLoadJob(ret[0]).get_state())) return palo_job.RoutineLoadJob(ret[0]).get_state() def wait_routine_load_state(self, routine_load_job_name, state='RUNNING', timeout=600, database_name=None): """ Args: routine_load_job_name: string, routine load job name state: string, 'NEED_SCHEDUL', 'PAUSE', 'RUNNING', 'STOPPED' timeout: int, timeout time Returns: """ while timeout > 0: job_state = self.get_routine_load_state(routine_load_job_name, database_name=database_name) if job_state == state: return True else: time.sleep(1) timeout -= 1 def show_tablet(self, table_name=None, database_name=None, tablet_id=None, partition_list=None): """ SHOW TABLETS [FROM [db_name.]table_name | tablet_id] [partiton(partition_name_1, partition_name_1)] [where [version=1] [and backendid=10000] [and state="NORMAL|ROLLUP|CLONE|DECOMMISSION"]] [order by order_column] [limit [offset,]size] """ if table_name is not None and tablet_id is not None: return None if table_name: if database_name is not None: table_name = '%s.%s' % (database_name, table_name) sql = 'SHOW TABLETS FROM %s' % table_name if partition_list: sql = '%s PARTITION (%s)' % (sql, ','.join(partition_list)) else: sql = 'SHOW TABLET %s' % tablet_id ret = self.execute(sql) return ret def explain_query(self, sql): """explain sql""" sql = 'EXPLAIN %s' % sql ret = self.execute(sql) return ret def show_txn(self, txn_id, database_name=None): """help show TRANSACTION for more msg""" if database_name is None: sql = "SHOW TRANSACTION WHERE id = %s" % txn_id else: sql = "SHOW TRANSACTION FROM %s WHERE id = %s" % (database_name, txn_id) ret = self.execute(sql) return ret def wait_txn(self, txn_id, database_name=None, status='VISIBLE', timeout=600): """wait txn util the status""" while timeout > 0: txn = self.show_txn(txn_id, database_name) txn_status = palo_job.TransactionInfo(txn[0]).get_transaction_status() if txn_status == status: LOG.info(L('GET TXN STATUS', txn_id=txn_id, status=txn_status)) return True elif txn_status == 'VISIBLE' or txn_status == 'ABORT': LOG.info(L('GET TXN STATUS', txn_id=txn_id, status=txn_status)) return False else: time.sleep(1) timeout -= 1 LOG.info(L('GET TXN STATUS TIMEOUT', txn_id=txn_id)) return False def create_bitmap_index_table(self, table_name, bitmap_index_name, index_column_name, storage_type=None, database_name=None, create_format=1, is_wait=False, cluster_name=None): """ Create a bitmap index """ database_name = self.database_name if database_name is None else database_name if create_format == 1: sql = 'ALTER TABLE %s.%s ADD INDEX %s (%s) USING BITMAP' % (database_name, table_name, \ bitmap_index_name, index_column_name) elif create_format == 2: sql = 'CREATE INDEX %s ON %s.%s (%s) USING BITMAP' % (bitmap_index_name, database_name, \ table_name, index_column_name) ret = self.execute(sql) if ret != (): LOG.info(L("CREATE BITMAP INDEX fail.", database_name=database_name, \ table_name=table_name, \ bitmap_index_name=bitmap_index_name)) return False ret = True if is_wait: ret = self.wait_table_schema_change_job(table_name, cluster_name=cluster_name, database_name=database_name) LOG.info(L("CREATE BITMAP INDEX succ.", database_name=database_name, \ table_name=table_name, \ bitmap_index_name=bitmap_index_name)) return ret def drop_bitmap_index_table(self, table_name, bitmap_index_name, storage_type=None, database_name=None, create_format=1, is_wait=False, cluster_name=None): """ Drop a bitmap index """ database_name = self.database_name if database_name is None else database_name if create_format == 1: sql = 'ALTER TABLE %s.%s DROP INDEX %s' % (database_name, table_name, bitmap_index_name) elif create_format == 2: sql = 'DROP INDEX %s ON %s.%s' % (bitmap_index_name, database_name, table_name) ret = self.execute(sql) if ret != (): LOG.info(L("DROP BITMAP INDEX fail.", database_name=database_name, \ table_name=table_name, \ bitmap_index_name=bitmap_index_name)) return False ret = True if is_wait: ret = self.wait_table_schema_change_job(table_name, cluster_name=cluster_name, database_name=database_name) LOG.info(L("DROP BITMAP INDEX succ.", database_name=database_name, \ table_name=table_name, \ bitmap_index_name=bitmap_index_name)) return ret def get_bitmap_index_list(self, table_name, database_name=None): """ Get index list from table """ database_name = self.database_name if database_name is None else database_name sql = "SHOW INDEX FROM %s.%s" % (database_name, table_name) ret = self.execute(sql) return ret def is_exists_index_in_table(self, index_name, index_name_col, table_name, database_name=None): """ return True if index exists in table, else return False """ database_name = self.database_name if database_name is None else database_name index_list = self.get_bitmap_index_list(table_name, database_name) if not index_list: return False for each_index_info in index_list: job_get_index = palo_job.TableIndexInfo(each_index_info) job_index_name = job_get_index.get_key_name() job_index_column = job_get_index.get_column_name() job_index_type = job_get_index.get_index_type() if index_name == job_index_name and index_name_col == job_index_column and \ job_index_type == "BITMAP": return True return False def select_into(self, query, output_file, broker, property=None, format_as=None): """ broker is BrokerInfo class property: dict, for csv, eg.{"column_separator": ",", "line_delimiter": "\n", "max_file_size": "100MB"} query_stmt INTO OUTFILE "file:///path/to/file_prefix" FORMAT AS CSV|PARQUET PROPERTIES (broker_propterties & other_properties); eg: SELECT * FROM tbl INTO OUTFILE "hdfs:/path/to/result_" FORMAT AS CSV PROPERTIELS ( "broker.name" = "my_broker", "broker.hadoop.security.authentication" = "kerberos", "broker.kerberos_principal" = "doris@YOUR.COM", "broker.kerberos_keytab" = "/home/doris/my.keytab" "column_separator" = ",", "line_delimiter" = "\n", "max_file_size" = "100MB" ); """ sql = '{query} INTO OUTFILE "{outfile}" {format_as} PROPERTIES ({properties}) ' if format_as is None: format_as = '' else: format_as = 'FORMAT AS %s' % format_as broker_property = broker.to_select_into_broker_property_str() if property is not None: into_properties = '' for k, v in property.items(): into_properties += ' "%s" = "%s",' % (k, v) p = into_properties.strip(',') property = broker_property + ',' + p else: property = broker_property sql = sql.format(query=query, outfile=output_file, format_as=format_as, properties=property) LOG.info(L('SELECT INTO.', sql=sql)) rows, ret = self.execute(sql, True) LOG.info(L('SELECT INTO ret.', ret=ret)) return ret def set_variables(self, k, v, is_global=False): """ set variables 如果为global需要重新connect，show的时候才会看到新的设置 """ if is_global: sql = 'SET GLOBAL %s=%s' % (k, v) else: sql = 'SET %s=%s' % (k, v) ret = self.execute(sql) return ret == () def show_variables(self, prefix=None): """ show variables """ if prefix: sql = 'SHOW VARIABLES LIKE "%%%s%%"' % prefix else: sql = 'SHOW VARIABLES' ret = self.execute(sql) return ret def wait_routine_load_commit(self, routine_load_job_name, committed_expect_num, timeout=600): """wait task committed""" print('expect commited rows: %s\n' % committed_expect_num) while timeout > 0: ret = self.show_routine_load(routine_load_job_name) routine_load_job = palo_job.RoutineLoadJob(ret[0]) loaded_rows = routine_load_job.get_loaded_rows() print(loaded_rows) if str(loaded_rows) == str(committed_expect_num): time.sleep(3) return True timeout -= 3 time.sleep(3) return False def enable_feature_batch_delete(self, table_name, database_name=None, is_wait=True): """enable feature batch delete""" if database_name is None: sql = 'ALTER TABLE %s ENABLE FEATURE "BATCH_DELETE"' % table_name else: sql = 'ALTER TABLE %s.%s ENABLE FEATURE "BATCH_DELETE"' % (database_name, table_name) ret = self.execute(sql) if is_wait: ret = self.wait_table_schema_change_job(table_name, database_name) return ret return ret == () def truncate(self, table_name, partition_list=None, database_name=None): """truncate table / partition""" if database_name is None: sql = 'TRUNCATE TABLE %s' % table_name else: sql = 'TRUNCATE TABLE %s.%s' % (database_name, table_name) if partition_list is not None: sql = '%s PARTITION (%s)' % (sql, ','.join(partition_list)) ret = self.execute(sql) return ret == () def commit(self): """commit""" ret = self.execute('COMMIT') return ret == () def begin(self): """begin""" ret = self.execute('BEGIN') return ret == () def rollback(self): """rollback""" ret = self.execute('ROLLBACK') return ret == () def update(self, table_name, set_list, where_clause=None, database_name=None): """ ref: UPDATE table_reference SET assignment_list [WHERE where_condition] table_name: str set_list: ['k1=2', 'k3=k3+1'] where_clause: str or ['k1 > 0'], 当是list的时候，使用and进行连接 """ if database_name is not None: table_name = '%s.%s' % (database_name, table_name) if isinstance(set_list, list): set_ref = ','.join(set_list) else: set_ref = set_list if where_clause is not None: if isinstance(where_clause, str): where_ref = 'WHERE %s' % where_clause elif isinstance(where_clause, list): where_ref = 'WHERE %s' % ' AND '.join(where_clause) # else: pass else: where_ref = '' sql = 'UPDATE {tb} SET {set_ref} {where_ref}'.format(tb=table_name, set_ref=set_ref, where_ref=where_ref) ret = self.execute(sql) return ret == () def admin_show_config(self, key=None): """ADMIN SHOW FRONTEND CONFIG [LIKE "pattern"]""" if key is None: sql = 'ADMIN SHOW FRONTEND CONFIG' else: sql = 'ADMIN SHOW FRONTEND CONFIG LIKE "%s"' % key return self.execute(sql) def show_dynamic_partition_tables(self, database_name=None): """ get dynamic partition table families """ if database_name is None: sql = "SHOW DYNAMIC PARTITION TABLES" result = self.execute(sql) return result sql = "SHOW DYNAMIC PARTITION TABLES FROM %s" % database_name result = self.execute(sql) return result def get_comment(self, database_name, table_name): """ get table comment """ sql = "select TABLE_COMMENT from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s'" \ % (database_name, table_name) ret = self.execute(sql) return ret[0][0] def get_column_comment(self, table_name, column_name): """ get column comment """ sql = "show full columns from %s" % table_name columns = self.execute(sql) for column in columns: if column[0] == column_name: return column[8] def create_sync_job(self, table_name, database_name, mysql_table_name, mysql_database_name, job_name, canal_ip, columns=None, partitions=None, canal_port='11111', destination='example', batch_size='8192', username='', password='', is_wait=True): """ create sync job table_name: str or list，导入到doris的表 eg: 'table_1', ['table_1', 'table_2'] database_name: str，binlog任务所在的doris数据库 mysql_table_name: str or list，导入的MySQL表，需与doris表一一对应 eg:['mysql_table_1', 'mysql_table_2'] mysql_database_name: list，导入的MySQL表的数据库 columns: list or None，指定列映射，eg: ['k1', 'k2'], [['k1', 'k2'], ['k2', 'v1']] partitions: list or None，指定导入分区，eg: ['p1', 'p2'], [[], ['p1', 'p3']] """ if not isinstance(table_name, list): table_name = [table_name, ] if not isinstance(mysql_table_name, list): mysql_table_name = [mysql_table_name, ] if not isinstance(mysql_database_name, list): mysql_database_name = [mysql_database_name, ] if columns is not None and not isinstance(columns[0], list): columns = [columns] if partitions is not None and not isinstance(partitions[0], list): partitions = [partitions] if len(table_name) != len(mysql_table_name): LOG.info(L('doris tables are not corresponding to mysql tables')) return False channel_desc = "" for i in range(len(table_name)): column_desc = '' if columns is not None and columns[i] is not []: for column in columns[i]: column_desc = '%s%s,' % (column_desc, column) column_desc = '(%s)' % column_desc[:-1] partition_desc = '' if partitions is not None and partitions[i] is not []: if len(partitions[i]) > 1: for partition in partitions[i]: partition_desc = '%s%s,' % (partition_desc, partition) partition_desc = 'PARTITIONS (%s)' % partition_desc[:-1] else: partition_desc = 'PARTITION (%s)' % partitions[i][0] else: partition_desc = '' channel_desc = "%s FROM %s.%s INTO %s %s %s," % (channel_desc, mysql_database_name[i], \ mysql_table_name[i], table_name[i], partition_desc, column_desc) sql = "CREATE SYNC %s.%s (%s)" \ "FROM BINLOG (" \ "'type' = 'canal'," \ "'canal.server.ip' = '%s'," \ "'canal.server.port' = '%s'," \ "'canal.destination' = '%s'," \ "'canal.batchSize' = '%s'," \ "'canal.username' = '%s'," \ "'canal.password' = '%s')" % (database_name, job_name, channel_desc[:-1], canal_ip, canal_port, \ destination, batch_size, username, password) ret = self.execute(sql) if ret != (): LOG.info(L('CREATE SYNC JOB fail.', job_name=job_name, database_name=database_name)) return False if is_wait: ret = self.wait_binlog_state(job_name) if not ret: LOG.info(L('CREATE SYNC JOB fail.', database_name=database_name, job_name=job_name)) return False LOG.info(L('CREATE SYNC JOB succ.', database_name=database_name, job_name=job_name)) return True def wait_binlog_state(self, job_name, state='RUNNING'): """ wait modify binlog job state """ job_state = self.get_sync_job_state(job_name) if job_state == state: LOG.info(L('SYNC JOB %s' % state)) return True timeout = 10 while timeout > 0: time.sleep(2) job_state = self.get_sync_job_state(job_name) if job_state == state: LOG.info(L('SYNC JOB %s' % state)) return True else: timeout -= 1 LOG.info(L('SYNC JOB STATE ERROR', EXPECTED=state, ACTUAL=job_state)) return False def pause_sync_job(self, job_name, database_name=None, is_wait=True): """ pause sync job """ if database_name is None: sql = 'PAUSE SYNC JOB %s' % job_name else: sql = 'PAUSE SYNC JOB %s.%s' % (database_name, job_name) ret = self.execute(sql) if ret != (): LOG.info(L('PAUSE SYNC JOB fail.', job_name=job_name, database_name=database_name)) return False if is_wait: ret = self.wait_binlog_state(job_name, 'PAUSED') if not ret: LOG.info(L('PAUSE SYNC JOB fail.', database_name=database_name, job_name=job_name)) return False LOG.info(L('PAUSE SYNC JOB succ.', database_name=database_name, job_name=job_name)) return True def resume_sync_job(self, job_name, database_name=None, is_wait=True): """ resume sync job """ if database_name is None: sql = 'RESUME SYNC JOB %s' % job_name else: sql = 'RESUME SYNC JOB %s.%s' % (database_name, job_name) ret = self.execute(sql) if ret != (): LOG.info(L('RESUME SYNC JOB fail.', job_name=job_name, database_name=database_name)) return False if is_wait: ret = self.wait_binlog_state(job_name) if not ret: LOG.info(L('RESUME SYNC JOB fail.', database_name=database_name, job_name=job_name)) return False LOG.info(L('RESUME SYNC JOB succ.', database_name=database_name, job_name=job_name)) return True def stop_sync_job(self, job_name, database_name=None, is_wait=True): """ stop sync job """ if database_name is None: sql = 'STOP SYNC JOB %s' % job_name else: sql = 'STOP SYNC JOB %s.%s' % (database_name, job_name) ret = self.execute(sql) if ret != (): LOG.info(L('STOP SYNC JOB fail.', job_name=job_name, database_name=database_name)) return False if is_wait: ret = self.wait_binlog_state(job_name, 'CANCELLED') if not ret: LOG.info(L('STOP SYNC JOB fail.', database_name=database_name, job_name=job_name)) return False LOG.info(L('STOP SYNC JOB succ.', database_name=database_name, job_name=job_name)) return True def get_sync_job_state(self, job_name): """ get sync job state """ sync_job_list = self.show_sync_job() condition_col_idx = palo_job.SyncJobInfo.JobName retrun_clo_idx = palo_job.SyncJobInfo.State return util.get_attr_condition_value(sync_job_list, condition_col_idx, job_name, retrun_clo_idx) def show_sync_job(self, database_name=None): """ get sync job information """ if database_name is None: sql = "SHOW SYNC JOB" result = self.execute(sql) return result sql = "SHOW SYNC JOB FROM %s" % database_name result = self.execute(sql) return result def set_frontend_config(self, config, value): """ admin set frontend config """ sql = 'ADMIN SET FRONTEND CONFIG ("%s" = "%s")' % (config, value) return self.execute(sql) def get_partition_replica_allocation(self, table_name, partition_name, database_name=None): """ get table family replica allocation """ partition = self.get_partition(table_name, partition_name, database_name) if partition: return partition[palo_job.PartitionInfo.ReplicaAllocation] else: return None def modify_resource_tag(self, host_name, port, tag_location): """ tag_location: str eg. grout_a, dict example: {'tag.location': 'a', 'tag.compute': 'b', ...} 修改be标签 """ if isinstance(tag_location, str): sql = "ALTER SYSTEM MODIFY BACKEND '%s:%s' SET ('tag.location'='%s')" % (host_name, port, tag_location) elif isinstance(tag_location, dict): sql = "ALTER SYSTEM MODIFY BACKEND '%s:%s' SET %s" % (host_name, port, util.convert_dict2property(tag_location)) else: return None result = self.execute(sql) time.sleep(2) if result != (): LOG.info(L('MODIFY BACKEND FAIL', backend=host_name, port=port, tag_location=tag_location)) return False return True def get_replica_backend_id(self, table_name): """get BackendId from replica status""" sql = "ADMIN SHOW REPLICA STATUS FROM %s" % table_name replica_status = self.execute(sql) column_idx = palo_job.ReplicaStatus.BackendId return util.get_attr(replica_status, column_idx) def admin_check_tablet(self, tablet_id_list): """admin check tablet""" sql = "ADMIN CHECK TABLE (%s) PROPERTIES('type'='consistency')" % ','.join(tablet_id_list) self.execute(sql) def admin_repair_table(self, table_name, partition_list=None): """admin repair table""" sql = "ADMIN REPAIR TABLE %s" % table_name if partition_list is not None: sql = '%s PARTITION (%s)' % (sql, ','.join(partition_list)) self.execute(sql) def admin_diagnose_tablet(self, tablet_id): """admin diagnose tablet""" sql = "ADMIN DIAGNOSE TABLET %s" % tablet_id ret = self.execute(sql) return ret def get_resource_tag(self, host_ip): """通过be的IP获取be的标签信息""" backend_list = self.get_backend_list() for backend in backend_list: be = palo_job.BackendProcInfo(backend) if be.get_ip() == host_ip: return be.get_tag() return None def get_resource_tag_by_id(self, be_id): """获取backend tag by id""" be = self.get_backend(be_id) if not be: return None return palo_job.BackendProcInfo(be).get_tag()

pytest/lib/palo_client.py (2,783 lines of code) (raw):