sql/dd/info_schema/stats.cc (533 lines of code) (raw):

/* Copyright (c) 2016, 2017 Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include "dd/info_schema/stats.h" // dd::info_schema::* #include <string.h> #include <cmath> #include <memory> // unique_ptr #include "auth_common.h" #include "dd/cache/dictionary_client.h" // dd::cache::Dictionary_client #include "dd/dd.h" // dd::create_object #include "dd/types/abstract_table.h" #include "dd/types/event.h" #include "dd/types/index_stat.h" // dd::Index_stat #include "dd/types/table_stat.h" // dd::Table_stat #include "debug_sync.h" // DEBUG_SYNC #include "error_handler.h" // Internal_error_handler #include "field.h" #include "key.h" #include "m_ctype.h" #include "mdl.h" #include "my_base.h" #include "my_dbug.h" #include "my_decimal.h" #include "my_sqlcommand.h" #include "my_sys.h" #include "my_time.h" // TIME_to_ulonglong_datetime #include "mysqld_error.h" #include "session_tracker.h" #include "sql_base.h" // open_tables_for_query #include "sql_class.h" // THD #include "sql_const.h" #include "sql_error.h" #include "sql_lex.h" #include "sql_list.h" #include "sql_security_ctx.h" #include "sql_show.h" // make_table_list #include "system_variables.h" #include "table.h" // TABLE_LIST #include "tztime.h" // Time_zone namespace dd { namespace info_schema { bool update_table_stats(THD *thd, TABLE_LIST *table) { // Update the object properties HA_CREATE_INFO create_info; memset(&create_info, 0, sizeof(create_info)); TABLE *analyze_table= table->table; handler *file= analyze_table->file; if (analyze_table->file->info(HA_STATUS_VARIABLE | HA_STATUS_TIME | HA_STATUS_VARIABLE_EXTRA | HA_STATUS_AUTO) != 0) return true; file->update_create_info(&create_info); // Create a object to be stored. std::unique_ptr<Table_stat> ts_obj(create_object<Table_stat>()); ts_obj->set_schema_name(String_type(table->db, strlen(table->db))); ts_obj->set_table_name(String_type(table->alias, strlen(table->alias))); ts_obj->set_table_rows(file->stats.records); ts_obj->set_avg_row_length(file->stats.mean_rec_length); ts_obj->set_data_length(file->stats.data_file_length); ts_obj->set_max_data_length(file->stats.max_data_file_length); ts_obj->set_index_length(file->stats.index_file_length); ts_obj->set_data_free(file->stats.delete_length); if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM) ts_obj->set_checksum(file->checksum()); MYSQL_TIME time; if (file->stats.update_time) { thd->variables.time_zone->gmt_sec_to_TIME(&time, (my_time_t) file->stats.update_time); ulonglong ull_time= TIME_to_ulonglong_datetime(&time); ts_obj->set_update_time(ull_time); } if (file->stats.check_time) { thd->variables.time_zone->gmt_sec_to_TIME(&time, (my_time_t) file->stats.check_time); ulonglong ull_time= TIME_to_ulonglong_datetime(&time); ts_obj->set_check_time(ull_time); } if (analyze_table->found_next_number_field) ts_obj->set_auto_increment(file->stats.auto_increment_value); else ts_obj->set_auto_increment(-1); // Store the object if (thd->dd_client()->store(ts_obj.get())) { my_error(ER_UNABLE_TO_STORE_STATISTICS, MYF(0), "table"); return true; } return false; } bool update_index_stats(THD *thd, TABLE_LIST *table) { // Update the object properties TABLE *analyze_table= table->table; KEY *key_info=analyze_table->s->key_info; if (analyze_table->file->info(HA_STATUS_VARIABLE | HA_STATUS_TIME | HA_STATUS_VARIABLE_EXTRA | HA_STATUS_AUTO) != 0) return true; // Create a object to be stored. std::unique_ptr<Index_stat> obj(create_object<Index_stat>()); for (uint i=0; i < analyze_table->s->keys; i++, key_info++) { KEY_PART_INFO *key_part= key_info->key_part; const char *str; ha_rows records; for (uint j=0 ; j < key_info->user_defined_key_parts ; j++,key_part++) { str=(key_part->field ? key_part->field->field_name : "?unknown field?"); KEY *key=analyze_table->key_info+i; if (key->has_records_per_key(j)) { double recs=(analyze_table->file->stats.records / key->records_per_key(j)); records= static_cast<longlong>(round(recs)); } else records= -1; // Treated as NULL obj->set_schema_name(String_type(table->db, strlen(table->db))); obj->set_table_name(String_type(table->alias, strlen(table->alias))); obj->set_index_name(String_type(key_info->name, strlen(key_info->name))); obj->set_column_name(String_type(str, strlen(str))); obj->set_cardinality((ulonglong) records); // Store the object if (thd->dd_client()->store(obj.get())) { my_error(ER_UNABLE_TO_STORE_STATISTICS, MYF(0), "index"); return true; } } // Key part info } // Keys return false; } // Convert IS db to lowercase and table case upper case. bool convert_table_name_case(char *db, char *table_name) { if (db && is_infoschema_db(db)) { my_casedn_str(system_charset_info, db); if (table_name && strncmp(table_name, "ndb", 3)) my_caseup_str(system_charset_info, table_name); return true; } return false; } /** Error handler class to convert ER_LOCK_DEADLOCK error to ER_WARN_I_S_SKIPPED_TABLE error. Handler is pushed for opening a table or acquiring a MDL lock on tables for INFORMATION_SCHEMA views(system views) operations. */ class MDL_deadlock_error_handler : public Internal_error_handler { public: MDL_deadlock_error_handler(THD *thd, const String *schema_name, const String *table_name) : m_can_deadlock(thd->mdl_context.has_locks()), m_schema_name(schema_name), m_table_name(table_name) {} virtual bool handle_condition(THD*, uint sql_errno, const char*, Sql_condition::enum_severity_level*, const char*) { if (sql_errno == ER_LOCK_DEADLOCK && m_can_deadlock) { // Convert error to ER_WARN_I_S_SKIPPED_TABLE. my_error(ER_WARN_I_S_SKIPPED_TABLE, MYF(0), m_schema_name->ptr(), m_table_name->ptr()); m_error_handled= true; } return false; } bool is_error_handled() const { return m_error_handled; } private: bool m_can_deadlock; // Schema name const String *m_schema_name; // Table name const String *m_table_name; // Flag to indicate whether deadlock error is handled by the handler or not. bool m_error_handled= false; }; // Returns the required statistics from the cache. ulonglong Statistics_cache::get_stat(ha_statistics &stat, enum_statistics_type stype) { switch (stype) { case enum_statistics_type::TABLE_ROWS: return(stat.records); case enum_statistics_type::TABLE_AVG_ROW_LENGTH: return(stat.mean_rec_length); case enum_statistics_type::DATA_LENGTH: return(stat.data_file_length); case enum_statistics_type::MAX_DATA_LENGTH: return(stat.max_data_file_length); case enum_statistics_type::INDEX_LENGTH: return(stat.index_file_length); case enum_statistics_type::DATA_FREE: return(stat.delete_length); case enum_statistics_type::AUTO_INCREMENT: return(stat.auto_increment_value); case enum_statistics_type::CHECKSUM: return(m_checksum); case enum_statistics_type::TABLE_UPDATE_TIME: return(stat.update_time); case enum_statistics_type::CHECK_TIME: return(stat.check_time); default: DBUG_ASSERT(!"Should not hit here"); } return 0; } // Read dynamic table statistics from SE by opening the user table // provided OR by reading cached statistics from SELECT_LEX. ulonglong Statistics_cache::read_stat( THD *thd, const String &schema_name_ptr, const String &table_name_ptr, const String &index_name_ptr, uint index_ordinal_position, uint column_ordinal_position, const String &engine_name_ptr, Object_id se_private_id, const char* ts_se_private_data, const char* tbl_se_private_data, enum_statistics_type stype) { DBUG_ENTER("Statistics_cache::read_stat"); ulonglong result; // NOTE: read_stat() may generate many "useless" warnings, which will be // ignored afterwards. On the other hand, there might be "useful" // warnings, which should be presented to the user. Diagnostics_area usually // stores no more than THD::variables.max_error_count warnings. // The problem is that "useless warnings" may occupy all the slots in the // Diagnostics_area, so "useful warnings" get rejected. In order to avoid // that problem we create a Diagnostics_area instance, which is capable of // storing "unlimited" number of warnings. Diagnostics_area *da= thd->get_stmt_da(); Diagnostics_area tmp_da(true); // Don't copy existing conditions from the old DA so we don't get them twice // when we call copy_non_errors_from_da below. thd->push_diagnostics_area(&tmp_da, false); /* If we have InnoDB table, then we try to get statistics without opening the table. */ if (!my_strcasecmp(system_charset_info, engine_name_ptr.ptr(), "InnoDB")) result= read_stat_from_SE(thd, schema_name_ptr, table_name_ptr, index_name_ptr, index_ordinal_position, column_ordinal_position, se_private_id, ts_se_private_data, tbl_se_private_data, stype); else result= read_stat_by_open_table(thd, schema_name_ptr, table_name_ptr, index_name_ptr, column_ordinal_position, stype); thd->pop_diagnostics_area(); // Pass an error if any. if (!thd->is_error() && tmp_da.is_error()) { da->set_error_status(tmp_da.mysql_errno(), tmp_da.message_text(), tmp_da.returned_sqlstate()); da->push_warning(thd, tmp_da.mysql_errno(), tmp_da.returned_sqlstate(), Sql_condition::SL_ERROR, tmp_da.message_text()); } // Pass warnings (if any). // // Filter out warnings with SL_ERROR level, because they // correspond to the errors which were filtered out in fill_table(). da->copy_non_errors_from_da(thd, &tmp_da); DBUG_RETURN(result); } // Fetch stats from SE ulonglong Statistics_cache::read_stat_from_SE( THD *thd, const String &schema_name_ptr, const String &table_name_ptr, const String &index_name_ptr, uint index_ordinal_position, uint column_ordinal_position, Object_id se_private_id, const char* ts_se_private_data, const char* tbl_se_private_data, enum_statistics_type stype) { DBUG_ENTER("Statistics_cache::read_stat_from_SE"); uint se_flags= 0; bool ignore_cache= false; ulonglong return_value= 0; // Stop we have see and error already for this table. if (check_error_for_key(schema_name_ptr, table_name_ptr)) DBUG_RETURN(0); /** It is faster to get first three statistics (below) alone when compared to getting them all. Also, Innodb does not give us check_time and checksum so we return from here. Notes for future, If there is a way to know which statistics have been requested in user query, then we can try to request SE with only those required statistics. E.g., A query requesting AUTO_INCREMENT and TABLE_ROWS both together would perform faster if we can combine HA_STATUS_AUTO | HA_STATUS_VARIABLE. Because optimizer silently removes unused internal UDF, we have no way to determine exactly what user had in the query. Currently if a user query requests just HA_STATUS_AUTO, it performance twice faster than requesting HA_STATUS_VARIABLE. So, for now we cache only HA_STATUS_VARIABLE, and skip cache for rest. */ switch (stype) { case enum_statistics_type::TABLE_UPDATE_TIME: se_flags= HA_STATUS_TIME; ignore_cache= true; break; case enum_statistics_type::DATA_FREE: se_flags= HA_STATUS_VARIABLE_EXTRA; ignore_cache= true; break; case enum_statistics_type::AUTO_INCREMENT: se_flags= HA_STATUS_AUTO; ignore_cache= true; break; case enum_statistics_type::CHECK_TIME: case enum_statistics_type::CHECKSUM: // InnoDB does return always zero for these statistics. DBUG_RETURN(0); case enum_statistics_type::INDEX_COLUMN_CARDINALITY: ignore_cache= true; break; default: se_flags= HA_STATUS_VARIABLE; } // // Get statistics from cache, if available // if (!ignore_cache && is_stat_cached(schema_name_ptr, table_name_ptr)) DBUG_RETURN(get_stat(stype)); // // Get statistics from InnoDB SE // ha_statistics ha_stat; // Build table name as required by InnoDB uint error= 0; handlerton *hton= ha_resolve_by_legacy_type(thd, DB_TYPE_INNODB); DBUG_ASSERT(hton); // InnoDB HA cannot be optional // Acquire MDL_EXPLICIT lock on table. MDL_request mdl_request; MDL_REQUEST_INIT(&mdl_request, MDL_key::TABLE, schema_name_ptr.ptr(), table_name_ptr.ptr(), MDL_SHARED_HIGH_PRIO, MDL_EXPLICIT); // Push deadlock error handler MDL_deadlock_error_handler mdl_deadlock_error_handler(thd, &schema_name_ptr, &table_name_ptr); thd->push_internal_handler(&mdl_deadlock_error_handler); if (thd->mdl_context.acquire_lock(&mdl_request, thd->variables.lock_wait_timeout)) error= -1; thd->pop_internal_handler(); DEBUG_SYNC(thd, "after_acquiring_mdl_shared_to_fetch_stats"); if (error == 0) { error= -1; /* It is possible that 'se_private_data' is not supplied to this function. The function dd::Properties::parse_properties() would at-least needs a single key-value pair to return a dd::Properties object. So, when se_private_data is not supplied, we force creation of dd::Properties object by passing a dummy=0 key-value pair. */ std::unique_ptr<dd::Properties> ts_se_private_data_obj( dd::Properties::parse_properties( ts_se_private_data?ts_se_private_data:"dummy=0;")); std::unique_ptr<dd::Properties> tbl_se_private_data_obj( dd::Properties::parse_properties( tbl_se_private_data?tbl_se_private_data:"dummy=0;")); // // Read statistics from SE // return_value= -1; if (stype == enum_statistics_type::INDEX_COLUMN_CARDINALITY && hton->get_index_column_cardinality && !hton->get_index_column_cardinality( schema_name_ptr.ptr(), table_name_ptr.ptr(), index_name_ptr.ptr(), index_ordinal_position, column_ordinal_position, se_private_id, &return_value)) { error= 0; } else if (hton->get_table_statistics && !hton->get_table_statistics(schema_name_ptr.ptr(), table_name_ptr.ptr(), se_private_id, *ts_se_private_data_obj.get(), *tbl_se_private_data_obj.get(), se_flags, &ha_stat)) { error= 0; } // Release the lock we got thd->mdl_context.release_lock(mdl_request.ticket); } // Cache and return the statistics if (error == 0) { if (!ignore_cache) cache_stats(schema_name_ptr, table_name_ptr, ha_stat); // Only cardinality is not stored in the cache. if (stype != enum_statistics_type::INDEX_COLUMN_CARDINALITY) return_value= get_stat(ha_stat, stype); DBUG_RETURN(return_value); } else if (thd->is_error()) { /* Hide error for a non-existing table. For example, this error can occur when we use a where condition with a db name and table, but the table does not exist. */ if (!(thd->get_stmt_da()->mysql_errno() == ER_NO_SUCH_TABLE) && !(thd->get_stmt_da()->mysql_errno() == ER_WRONG_OBJECT)) push_warning(thd, Sql_condition::SL_WARNING, thd->get_stmt_da()->mysql_errno(), thd->get_stmt_da()->message_text()); /* Cache empty statistics when we see a error. This will make sure, 1. You will not invoke open_tables_for_query() gain. 2. You will not see junk values for statistics in results. */ cache_stats(schema_name_ptr, table_name_ptr, ha_stat); m_error= thd->get_stmt_da()->message_text(); thd->clear_error(); } DBUG_RETURN(error); } // Fetch stats by opening the table. ulonglong Statistics_cache::read_stat_by_open_table( THD *thd, const String &schema_name_ptr, const String &table_name_ptr, const String &index_name_ptr, uint column_ordinal_position, enum_statistics_type stype) { DBUG_ENTER("Statistics_cache::read_stat_by_open_table"); ulonglong return_value= 0; uint error= 0; ha_statistics ha_stat; // // Get statistics from cache, if available // if (check_error_for_key(schema_name_ptr, table_name_ptr)) DBUG_RETURN(0); if (stype != enum_statistics_type::INDEX_COLUMN_CARDINALITY && is_stat_cached(schema_name_ptr, table_name_ptr)) DBUG_RETURN(get_stat(stype)); // // Get statistics by opening the table // MDL_deadlock_error_handler mdl_deadlock_error_handler(thd, &schema_name_ptr, &table_name_ptr); Open_tables_backup open_tables_state_backup; thd->reset_n_backup_open_tables_state(&open_tables_state_backup, 0); Query_arena i_s_arena(thd->mem_root, Query_arena::STMT_CONVENTIONAL_EXECUTION); Query_arena *old_arena= thd->stmt_arena; thd->stmt_arena= &i_s_arena; Query_arena backup_arena; thd->set_n_backup_active_arena(&i_s_arena, &backup_arena); LEX temp_lex, *lex; LEX *old_lex= thd->lex; thd->lex= lex= &temp_lex; lex_start(thd); lex->context_analysis_only= CONTEXT_ANALYSIS_ONLY_VIEW; LEX_CSTRING db_name_lex_cstr, table_name_lex_cstr; if (!thd->make_lex_string(&db_name_lex_cstr, schema_name_ptr.ptr(), schema_name_ptr.length(), FALSE) || !thd->make_lex_string(&table_name_lex_cstr, table_name_ptr.ptr(), table_name_ptr.length(), FALSE)) { error= -1; goto end; } if (make_table_list(thd, lex->select_lex, db_name_lex_cstr, table_name_lex_cstr)) { error= -1; goto end; } TABLE_LIST *table_list; table_list= lex->select_lex->table_list.first; table_list->required_type= dd::enum_table_type::BASE_TABLE; /* Let us set fake sql_command so views won't try to merge themselves into main statement. If we don't do this, SELECT * from information_schema.xxxx will cause problems. SQLCOM_SHOW_FIELDS is used because it satisfies 'only_view_structure()'. */ lex->sql_command= SQLCOM_SELECT; DBUG_EXECUTE_IF("simulate_kill_query_on_open_table", DBUG_SET("+d,kill_query_on_open_table_from_tz_find");); // Push deadlock error handler. thd->push_internal_handler(&mdl_deadlock_error_handler); bool open_result; open_result= open_tables_for_query(thd, table_list, MYSQL_OPEN_IGNORE_FLUSH | MYSQL_OPEN_FORCE_SHARED_HIGH_PRIO_MDL); thd->pop_internal_handler(); DBUG_EXECUTE_IF("simulate_kill_query_on_open_table", DBUG_SET("-d,kill_query_on_open_table_from_tz_find");); DEBUG_SYNC(thd, "after_open_table_mdl_shared_to_fetch_stats"); if (!open_result && table_list->is_view_or_derived()) { open_result= table_list->resolve_derived(thd, false); if (!open_result) open_result= table_list->setup_materialized_derived(thd); } /* Restore old value of sql_command back as it is being looked at in process_table() function. */ lex->sql_command= old_lex->sql_command; if (open_result) { DBUG_ASSERT(thd->is_error() || thd->is_killed()); if (thd->is_error()) { /* Hide error for a non-existing table. For example, this error can occur when we use a where condition with a db name and table, but the table does not exist. */ if (!(thd->get_stmt_da()->mysql_errno() == ER_NO_SUCH_TABLE) && !(thd->get_stmt_da()->mysql_errno() == ER_WRONG_OBJECT)) push_warning(thd, Sql_condition::SL_WARNING, thd->get_stmt_da()->mysql_errno(), thd->get_stmt_da()->message_text()); /* Cache empty statistics when we see a error. This will make sure, 1. You will not invoke open_tables_for_query() gain. 2. You will not see junk values for statistics in results. */ cache_stats(schema_name_ptr, table_name_ptr, ha_stat); m_error= thd->get_stmt_da()->message_text(); thd->clear_error(); } else { /* Table open fails even when query or connection is killed. In this case Diagnostics_area might not be set. So just returning error from here. Query is later terminated by call to send_kill_message() when we check thd->killed flag. */ error= -1; } goto end; } else if (!table_list->is_view() && !table_list->schema_table) { if (table_list->table->file->info(HA_STATUS_VARIABLE | HA_STATUS_TIME | HA_STATUS_VARIABLE_EXTRA | HA_STATUS_AUTO) != 0) { if (thd->is_error()) { push_warning(thd, Sql_condition::SL_WARNING, thd->get_stmt_da()->mysql_errno(), thd->get_stmt_da()->message_text()); /* Cache empty statistics when we see a error. This will make sure, 1. You will not invoke open_tables_for_query() gain. 2. You will not see junk values for statistics in results. */ cache_stats(schema_name_ptr, table_name_ptr, ha_stat); m_error= thd->get_stmt_da()->message_text(); thd->clear_error(); } else error= -1; goto end; } // If we are reading cardinality, just read and do not cache it. if (stype == enum_statistics_type::INDEX_COLUMN_CARDINALITY) { TABLE *table= table_list->table; uint key_index= 0; // Search for key with the index name. while (key_index < table->s->keys) { if (!my_strcasecmp(system_charset_info, (table->key_info+key_index)->name, index_name_ptr.ptr())) break; key_index++; } KEY *key= table->s->key_info + key_index; // Calculate the cardinality. ha_rows records; if (key_index < table->s->keys && key->has_records_per_key(column_ordinal_position)) { records=(table->file->stats.records / key->records_per_key(column_ordinal_position)); records= static_cast<longlong>(round(records)); } else records= -1; // Treated as NULL return_value= (ulonglong) records; } else // Get all statistics and cache them. { cache_stats(schema_name_ptr, table_name_ptr, table_list->table->file); return_value= get_stat(stype); } } else { error= -1; goto end; } end: lex->unit->cleanup(true); /* Restore original LEX value, statement's arena and THD arena values. */ lex_end(thd->lex); // Free items, before restoring backup_arena below. DBUG_ASSERT(i_s_arena.free_list == NULL); thd->free_items(); /* For safety reset list of open temporary tables before closing all tables open within this Open_tables_state. */ close_thread_tables(thd); /* Release metadata lock we might have acquired. See comment in fill_schema_table_from_frm() for details. */ thd->mdl_context.rollback_to_savepoint(open_tables_state_backup.mdl_system_tables_svp); thd->lex= old_lex; thd->stmt_arena= old_arena; thd->restore_active_arena(&i_s_arena, &backup_arena); thd->restore_backup_open_tables_state(&open_tables_state_backup); /* ER_LOCK_DEADLOCK is converted to ER_WARN_I_S_SKIPPED_TABLE by deadlock error handler used here. If rollback request is set by other deadlock error handlers then reset it here. */ if (mdl_deadlock_error_handler.is_error_handled() && thd->transaction_rollback_request) thd->transaction_rollback_request= false; DBUG_RETURN(error==0 ? return_value : error); } } }