in phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java [4124:4500]
protected PhoenixConnection upgradeSystemCatalogIfRequired(PhoenixConnection metaConnection,
long currentServerSideTableTimeStamp) throws SQLException, IOException, TimeoutException, InterruptedException {
String columnsToAdd = "";
// This will occur if we have an older SYSTEM.CATALOG and we need to update it to
// include any new columns we've added.
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
// We know that we always need to add the STORE_NULLS column for 4.3 release
columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.STORE_NULLS
+ " " + PBoolean.INSTANCE.getSqlTypeName());
try (Admin admin = getAdmin()) {
List<TableDescriptor> localIndexTables =
admin.listTableDescriptors(Pattern
.compile(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + ".*"));
for (TableDescriptor table : localIndexTables) {
if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null
&& table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
table=TableDescriptorBuilder.newBuilder(table).setValue(Bytes.toBytes(MetaDataUtil.PARENT_TABLE_KEY),
Bytes.toBytes(MetaDataUtil.getLocalIndexUserTableName(table.getTableName().getNameAsString()))).build();
// Explicitly disable, modify and enable the table to ensure
// co-location of data and index regions. If we just modify the
// table descriptor when online schema change enabled may reopen
// the region in same region server instead of following data region.
disableTable(admin, table.getTableName());
admin.modifyTable(table);
admin.enableTable(table.getTableName());
}
}
}
}
// If the server side schema is before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then
// we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too.
// TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed,
// we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all
// the column names that have been added to SYSTEM.CATALOG since 4.0.
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) {
columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.INDEX_TYPE + " "
+ PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", "
+ PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " "
+ PLong.INSTANCE.getSqlTypeName());
}
// If we have some new columns from 4.1-4.3 to add, add them now.
if (!columnsToAdd.isEmpty()) {
// Ugh..need to assign to another local variable to keep eclipse happy.
PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0, columnsToAdd);
metaConnection = newMetaConnection;
}
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0) {
columnsToAdd = PhoenixDatabaseMetaData.BASE_COLUMN_COUNT + " "
+ PInteger.INSTANCE.getSqlTypeName();
try {
metaConnection = addColumn(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0, columnsToAdd,
false);
upgradeTo4_5_0(metaConnection);
} catch (ColumnAlreadyExistsException ignored) {
/*
* Upgrade to 4.5 is a slightly special case. We use the fact that the
* column BASE_COLUMN_COUNT is already part of the meta-data schema as the
* signal that the server side upgrade has finished or is in progress.
*/
LOGGER.debug("No need to run 4.5 upgrade");
}
Properties p = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
p.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB);
p.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
PhoenixConnection conn = new PhoenixConnection(
ConnectionQueryServicesImpl.this, metaConnection.getURL(), p);
try {
List<String> tablesNeedingUpgrade = UpgradeUtil
.getPhysicalTablesWithDescRowKey(conn);
if (!tablesNeedingUpgrade.isEmpty()) {
LOGGER.warn("The following tables require upgrade due to a bug " +
"causing the row key to be incorrect for descending columns " +
"and ascending BINARY columns (PHOENIX-2067 and PHOENIX-2120):\n"
+ Joiner.on(' ').join(tablesNeedingUpgrade)
+ "\nTo upgrade issue the \"bin/psql.py -u\" command.");
}
List<String> unsupportedTables = UpgradeUtil
.getPhysicalTablesWithDescVarbinaryRowKey(conn);
if (!unsupportedTables.isEmpty()) {
LOGGER.warn("The following tables use an unsupported " +
"VARBINARY DESC construct and need to be changed:\n"
+ Joiner.on(' ').join(unsupportedTables));
}
} catch (Exception ex) {
LOGGER.error(
"Unable to determine tables requiring upgrade due to PHOENIX-2067",
ex);
} finally {
conn.close();
}
}
// Add these columns one at a time so that if folks have run the upgrade code
// already for a snapshot, we'll still enter this block (and do the parts we
// haven't yet done).
// Add each column with different timestamp else the code assumes that the
// table is already modified at that timestamp resulting in not updating the
// second column with same timestamp
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) {
columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " "
+ PBoolean.INSTANCE.getSqlTypeName();
metaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd);
}
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
// Drop old stats table so that new stats table is created
metaConnection = dropStatsTable(metaConnection,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4);
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3,
PhoenixDatabaseMetaData.TRANSACTIONAL + " "
+ PBoolean.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " "
+ PLong.INSTANCE.getSqlTypeName());
metaConnection = setImmutableTableIndexesImmutable(metaConnection,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
metaConnection = updateSystemCatalogTimestamp(metaConnection,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
ConnectionQueryServicesImpl.this.removeTable(null,
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
clearCache();
}
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0) {
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 2,
PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED + " "
+ PBoolean.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 1,
PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ + " "
+ PVarchar.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0,
PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA + " "
+ PBoolean.INSTANCE.getSqlTypeName());
metaConnection = UpgradeUtil.disableViewIndexes(metaConnection);
if (getProps().getBoolean(QueryServices.LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB,
QueryServicesOptions.DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE)) {
localIndexUpgradeRequired = true;
}
ConnectionQueryServicesImpl.this.removeTable(null,
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
clearCache();
}
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " "
+ PLong.INSTANCE.getSqlTypeName());
ConnectionQueryServicesImpl.this.removeTable(null,
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
clearCache();
}
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
metaConnection = addColumnQualifierColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 3);
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2,
PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME + " "
+ PTinyint.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 1,
PhoenixDatabaseMetaData.ENCODING_SCHEME + " "
+ PTinyint.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0,
PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER + " "
+ PInteger.INSTANCE.getSqlTypeName());
ConnectionQueryServicesImpl.this.removeTable(null,
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0);
clearCache();
}
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0,
PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION + " "
+ PBoolean.INSTANCE.getSqlTypeName());
addParentToChildLinks(metaConnection);
}
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
+ PTinyint.INSTANCE.getSqlTypeName());
try (Statement altQry = metaConnection.createStatement()) {
altQry.executeUpdate("ALTER TABLE "
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG + " SET "
+ HConstants.VERSIONS + "= "
+ props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions
.DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n"
+ ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "="
+ props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB,
QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
altQry.executeUpdate("ALTER TABLE "
+ PhoenixDatabaseMetaData.SYSTEM_FUNCTION + " SET "
+ TableDescriptorBuilder.SPLIT_POLICY + "='"
+ QueryConstants.SYSTEM_FUNCTION_SPLIT_POLICY_CLASSNAME + "',\n"
+ HConstants.VERSIONS + "= "
+ props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions
.DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n"
+ ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "="
+ props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB,
QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
altQry.executeUpdate("ALTER TABLE "
+ PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET "
+ TableDescriptorBuilder.SPLIT_POLICY + "='"
+ QueryConstants.SYSTEM_STATS_SPLIT_POLICY_CLASSNAME + "'");
}
}
if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP_4_15_0) {
addViewIndexToParentLinks(metaConnection);
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_4_15_0,
PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE + " "
+ PInteger.INSTANCE.getSqlTypeName());
}
if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0) {
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0 - 3,
PhoenixDatabaseMetaData.PHOENIX_TTL + " "
+ PInteger.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(
metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0 - 2,
PhoenixDatabaseMetaData.PHOENIX_TTL_HWM + " "
+ PInteger.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG, MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0 -1,
PhoenixDatabaseMetaData.LAST_DDL_TIMESTAMP + " "
+ PLong.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG, MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0,
PhoenixDatabaseMetaData.CHANGE_DETECTION_ENABLED
+ " " + PBoolean.INSTANCE.getSqlTypeName());
UpgradeUtil.bootstrapLastDDLTimestampForTablesAndViews(metaConnection);
boolean isNamespaceMapping =
SchemaUtil.isNamespaceMappingEnabled(null, getConfiguration());
String tableName = PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
if (isNamespaceMapping) {
tableName = tableName.replace(
QueryConstants.NAME_SEPARATOR,
QueryConstants.NAMESPACE_SEPARATOR);
}
byte[] tableBytes = StringUtil.toBytes(tableName);
byte[] rowKey = SchemaUtil.getColumnKey(null,
QueryConstants.SYSTEM_SCHEMA_NAME,
SYSTEM_CATALOG_TABLE, VIEW_INDEX_ID,
PhoenixDatabaseMetaData.TABLE_FAMILY);
if (UpgradeUtil.isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded
(metaConnection, rowKey, tableBytes)) {
LOGGER.info("Updating VIEW_INDEX_ID data type to BIGINT.");
UpgradeUtil.updateViewIndexIdColumnDataTypeFromShortToLong(
metaConnection, rowKey, tableBytes);
} else {
LOGGER.info("Updating VIEW_INDEX_ID data type is not needed.");
}
try (Admin admin = metaConnection.getQueryServices().getAdmin()) {
TableDescriptorBuilder tdBuilder;
TableName sysCatPhysicalTableName = SchemaUtil.getPhysicalTableName(
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, props);
tdBuilder = TableDescriptorBuilder.newBuilder(
admin.getDescriptor(sysCatPhysicalTableName));
if (!tdBuilder.build().hasCoprocessor(
QueryConstants.SYSTEM_CATALOG_REGION_OBSERVER_CLASSNAME)) {
int priority = props.getInt(
QueryServices.COPROCESSOR_PRIORITY_ATTRIB,
QueryServicesOptions.DEFAULT_COPROCESSOR_PRIORITY);
tdBuilder.setCoprocessor(
CoprocessorDescriptorBuilder
.newBuilder(QueryConstants.SYSTEM_CATALOG_REGION_OBSERVER_CLASSNAME)
.setPriority(priority)
.setProperties(Collections.emptyMap())
.build());
admin.modifyTable(tdBuilder.build());
pollForUpdatedTableDescriptor(admin, tdBuilder.build(),
sysCatPhysicalTableName.getName());
}
}
}
if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0) {
metaConnection =
addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 7,
PhoenixDatabaseMetaData.PHYSICAL_TABLE_NAME + " "
+ PVarchar.INSTANCE.getSqlTypeName());
metaConnection =
addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 6,
PhoenixDatabaseMetaData.SCHEMA_VERSION + " "
+ PVarchar.INSTANCE.getSqlTypeName());
metaConnection =
addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 5,
PhoenixDatabaseMetaData.EXTERNAL_SCHEMA_ID + " "
+ PVarchar.INSTANCE.getSqlTypeName());
metaConnection =
addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 4,
PhoenixDatabaseMetaData.STREAMING_TOPIC_NAME + " "
+ PVarchar.INSTANCE.getSqlTypeName());
metaConnection =
addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 3,
PhoenixDatabaseMetaData.INDEX_WHERE + " "
+ PVarchar.INSTANCE.getSqlTypeName());
metaConnection =
addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 2,
PhoenixDatabaseMetaData.CDC_INCLUDE_TABLE + " "
+ PVarchar.INSTANCE.getSqlTypeName());
/**
* TODO: Provide a path to copy existing data from PHOENIX_TTL to TTL column and then
* to DROP PHOENIX_TTL Column. See PHOENIX-7023
*/
metaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG, MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 1,
PhoenixDatabaseMetaData.TTL + " " + PInteger.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG, MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0,
PhoenixDatabaseMetaData.ROW_KEY_MATCHER + " "
+ PVarbinary.INSTANCE.getSqlTypeName());
//Values in PHOENIX_TTL column will not be used for further release as PHOENIX_TTL column is being deprecated
//and will be removed in later release. To copy copyDataFromPhoenixTTLtoTTL(metaConnection) can be used but
//as that feature was not fully built we are not moving old value to new column
//move TTL values stored in descriptor to SYSCAT TTL column.
moveTTLFromHBaseLevelTTLToPhoenixLevelTTL(metaConnection);
UpgradeUtil.bootstrapLastDDLTimestampForTablesAndViews(metaConnection);
UpgradeUtil.bootstrapLastDDLTimestampForIndexes(metaConnection);
}
return metaConnection;
}