in asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java [1323:1676]
protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexStatement stmtCreateIndex,
String databaseName, DataverseName dataverseName, String datasetName, IHyracksClientConnection hcc,
IRequestParameters requestParameters, Creator creator) throws Exception {
SourceLocation sourceLoc = stmtCreateIndex.getSourceLocation();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
// Check if the dataverse exists
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, databaseName, dataverseName);
if (dv == null) {
throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc,
MetadataUtil.dataverseName(databaseName, dataverseName, metadataProvider.isUsingDatabase()));
}
Dataset ds = metadataProvider.findDataset(databaseName, dataverseName, datasetName);
if (ds == null) {
throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, sourceLoc, datasetName,
MetadataUtil.dataverseName(databaseName, dataverseName, metadataProvider.isUsingDatabase()));
}
DatasetType datasetType = ds.getDatasetType();
IndexType indexType = stmtCreateIndex.getIndexType();
List<CreateIndexStatement.IndexedElement> indexedElements = stmtCreateIndex.getIndexedElements();
int indexedElementsCount = indexedElements.size();
boolean isSecondaryPrimary = indexedElementsCount == 0;
validateIndexType(datasetType, indexType, isSecondaryPrimary, sourceLoc);
String indexName = stmtCreateIndex.getIndexName().getValue();
Index index = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), databaseName,
dataverseName, datasetName, indexName);
if (index != null) {
if (stmtCreateIndex.getIfNotExists()) {
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
return;
} else {
throw new CompilationException(ErrorCode.INDEX_EXISTS, sourceLoc, indexName);
}
}
Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
ds.getItemTypeDatabaseName(), ds.getItemTypeDataverseName(), ds.getItemTypeName());
ARecordType aRecordType = (ARecordType) dt.getDatatype();
/* TODO: unused for now becase indexes on meta are disabled -- see below
ARecordType metaRecordType = null;
if (ds.hasMetaPart()) {
Datatype metaDt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
metaRecordType = (ARecordType) metaDt.getDatatype();
}
*/
if (!ds.hasMetaPart()) {
aRecordType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(aRecordType, null, ds);
}
List<List<IAType>> indexFieldTypes = new ArrayList<>(indexedElementsCount);
boolean hadUnnest = false;
boolean overridesFieldTypes = false;
boolean isHeterogeneousIndex = false;
// this set is used to detect duplicates in the specified keys in the create
// index statement
// e.g. CREATE INDEX someIdx on dataset(id,id).
// checking only the names is not enough.
// Need also to check the source indicators for the most general case
// (even though indexes on meta fields are curently disabled -- see below)
Set<Triple<Integer, List<List<String>>, List<List<String>>>> indexKeysSet = new HashSet<>();
for (CreateIndexStatement.IndexedElement indexedElement : indexedElements) {
// disable creating an index on meta fields (fields with source indicator == 1 are meta fields)
if (indexedElement.getSourceIndicator() != Index.RECORD_INDICATOR) {
throw new AsterixException(ErrorCode.COMPILATION_ERROR, indexedElement.getSourceLocation(),
"Cannot create index on meta fields");
}
ARecordType sourceRecordType = aRecordType;
IAType inputTypePrime;
boolean inputTypeNullable, inputTypeMissable;
List<Pair<List<String>, IndexedTypeExpression>> projectList = indexedElement.getProjectList();
int projectCount = projectList.size();
if (indexedElement.hasUnnest()) {
if (indexType != IndexType.ARRAY) {
throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_INDEX_TYPE,
indexedElement.getSourceLocation(), String.valueOf(indexType));
}
// allow only 1 unnesting element in ARRAY index
if (hadUnnest) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, indexedElement.getSourceLocation(),
"Cannot create composite index with multiple array fields using different arrays");
}
hadUnnest = true;
if (projectCount == 0) {
// Note. UNNEST with no SELECT is supposed to have 1 project element with 'null' path
throw new CompilationException(ErrorCode.COMPILATION_ERROR, indexedElement.getSourceLocation(),
"Invalid index element");
}
Triple<IAType, Boolean, Boolean> unnestTypeResult = KeyFieldTypeUtil.getKeyUnnestType(
sourceRecordType, indexedElement.getUnnestList(), indexedElement.getSourceLocation());
if (unnestTypeResult == null) {
inputTypePrime = null; // = ANY
inputTypeNullable = inputTypeMissable = true;
} else {
inputTypePrime = unnestTypeResult.first;
inputTypeNullable = unnestTypeResult.second;
inputTypeMissable = unnestTypeResult.third;
}
} else {
if (projectCount != 1) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, indexedElement.getSourceLocation(),
"Invalid index element");
}
inputTypePrime = sourceRecordType;
inputTypeNullable = inputTypeMissable = false;
}
// at this point 'inputTypePrime' is either a record, or if we had unnest then it could also be anything else.
List<IAType> fieldTypes = new ArrayList<>(projectCount);
for (int i = 0; i < projectCount; i++) {
Pair<List<String>, IndexedTypeExpression> projectPair = projectList.get(i);
List<String> projectPath = projectPair.first;
IndexedTypeExpression projectTypeExpr = projectPair.second;
IAType projectTypePrime;
boolean projectTypeNullable, projectTypeMissable;
if (projectPath == null) {
boolean emptyPathOk = indexedElement.hasUnnest() && i == 0;
if (!emptyPathOk) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
indexedElement.getSourceLocation(), "Invalid index element");
}
projectTypePrime = inputTypePrime;
projectTypeNullable = inputTypeNullable;
projectTypeMissable = inputTypeMissable;
} else if (inputTypePrime == null) {
projectTypePrime = null;
projectTypeNullable = projectTypeMissable = true;
} else {
if (inputTypePrime.getTypeTag() != ATypeTag.OBJECT) {
throw new CompilationException(ErrorCode.TYPE_MISMATCH_GENERIC, sourceLoc, ATypeTag.OBJECT,
inputTypePrime.getTypeTag());
}
ARecordType inputTypePrimeRecord = (ARecordType) inputTypePrime;
Triple<IAType, Boolean, Boolean> projectTypeResult = KeyFieldTypeUtil.getKeyProjectType(
inputTypePrimeRecord, projectPath, indexedElement.getSourceLocation());
if (projectTypeResult != null) {
projectTypePrime = projectTypeResult.first;
projectTypeNullable = inputTypeNullable || projectTypeResult.second;
projectTypeMissable = inputTypeMissable || projectTypeResult.third;
} else {
projectTypePrime = null; // ANY
projectTypeNullable = projectTypeMissable = true;
}
}
boolean isFieldFromSchema = projectTypePrime != null;
IAType fieldTypePrime;
boolean fieldTypeNullable, fieldTypeMissable;
if (projectTypeExpr == null) {
// the type of the indexed field is NOT specified in the DDL
if (stmtCreateIndex.hasCastDefaultNull()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
stmtCreateIndex.getSourceLocation(),
"CAST modifier is used without specifying the type of the indexed field");
}
fieldTypePrime = projectTypePrime;
fieldTypeNullable = projectTypeNullable;
fieldTypeMissable = projectTypeMissable;
} else {
// the type of the indexed field is explicitly specified in the DDL
Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(databaseName, dataverseName,
indexName, projectTypeExpr.getType(), databaseName, dataverseName, mdTxnCtx);
TypeSignature typeSignature = new TypeSignature(databaseName, dataverseName, indexName);
fieldTypePrime = typeMap.get(typeSignature);
// BACK-COMPAT: keep prime type only if we're overriding field types
fieldTypeNullable = fieldTypeMissable = false;
overridesFieldTypes = true;
if (stmtCreateIndex.isEnforced()) {
if (!projectTypeExpr.isUnknownable()) {
throw new CompilationException(ErrorCode.INDEX_ILLEGAL_ENFORCED_NON_OPTIONAL,
indexedElement.getSourceLocation(),
LogRedactionUtil.userData(String.valueOf(projectPath)));
}
// don't allow creating an enforced index on a closed-type field having field type different
// from the closed-type
if (isFieldFromSchema) {
if (fieldTypePrime == null) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE,
indexedElement.getSourceLocation(), "cannot find type of field");
} else if (!projectTypePrime.deepEqual(fieldTypePrime)) {
throw new CompilationException(ErrorCode.TYPE_MISMATCH_GENERIC, sourceLoc,
projectTypePrime.getTypeTag(), fieldTypePrime.getTypeTag());
}
}
} else {
if (indexType != IndexType.BTREE && indexType != IndexType.ARRAY) {
throw new CompilationException(ErrorCode.INDEX_ILLEGAL_NON_ENFORCED_TYPED,
indexedElement.getSourceLocation(), indexType);
}
if (isFieldFromSchema) {
// allow overriding the type of the closed-field only if CAST modifier is used
if (!stmtCreateIndex.hasCastDefaultNull()) {
if (fieldTypePrime == null) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE,
indexedElement.getSourceLocation(), "cannot find type of field");
} else if (!projectTypePrime.deepEqual(fieldTypePrime)) {
throw new CompilationException(ErrorCode.TYPE_MISMATCH_GENERIC, sourceLoc,
projectTypePrime.getTypeTag(), fieldTypePrime.getTypeTag());
}
}
}
}
}
if (fieldTypePrime == null) {
if (indexType != IndexType.BTREE) {
if (projectPath != null) {
String fieldName =
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(projectPath));
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
indexedElement.getSourceLocation(),
"cannot find type of field '" + fieldName + "'");
}
// projectPath == null should only be the case with array index having UNNESTs only
if (indexedElement.hasUnnest()) {
List<List<String>> unnestList = indexedElement.getUnnestList();
List<String> arrayField = unnestList.get(unnestList.size() - 1);
String fieldName =
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(arrayField));
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
indexedElement.getSourceLocation(),
"cannot find type of elements of field '" + fieldName + "'");
}
} else {
fieldTypePrime = BuiltinType.ANY;
isHeterogeneousIndex = true;
fieldTypeNullable = fieldTypeMissable = false;
}
}
if (ATypeTag.ANY.equals(fieldTypePrime.getTypeTag()) && stmtCreateIndex.hasExcludeUnknownKey()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, indexedElement.getSourceLocation(),
"Cannot specify exclude/include unknown for untyped keys in the index definition.");
}
validateIndexFieldType(indexType, fieldTypePrime, projectPath, indexedElement.getSourceLocation(),
isHeterogeneousIndex);
IAType fieldType =
KeyFieldTypeUtil.makeUnknownableType(fieldTypePrime, fieldTypeNullable, fieldTypeMissable);
if (isHeterogeneousIndex && !ATypeTag.ANY.equals(fieldType.getTypeTag())) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Typed keys cannot be combined with untyped keys in the index definition.");
}
fieldTypes.add(fieldType);
}
// Try to add the key & its source to the set of keys for duplicate detection.
if (!indexKeysSet.add(indexedElement.toIdentifier())) {
throw new AsterixException(ErrorCode.INDEX_ILLEGAL_REPETITIVE_FIELD,
indexedElement.getSourceLocation(),
LogRedactionUtil.userData(indexedElement.getProjectListDisplayForm()));
}
indexFieldTypes.add(fieldTypes);
}
boolean unknownKeyOptionAllowed =
(indexType == IndexType.BTREE || indexType == IndexType.ARRAY) && !isSecondaryPrimary;
if (stmtCreateIndex.hasExcludeUnknownKey() && !unknownKeyOptionAllowed) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"can only specify exclude/include unknown key for B-Tree & Array indexes");
}
boolean castDefaultNullAllowed = indexType == IndexType.BTREE && !isSecondaryPrimary;
if (stmtCreateIndex.hasCastDefaultNull() && !castDefaultNullAllowed) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"CAST modifier is only allowed for B-Tree indexes");
}
if (stmtCreateIndex.getCastDefaultNull().getOrElse(false)) {
if (stmtCreateIndex.isEnforced()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"CAST modifier cannot be specified together with ENFORCED");
}
}
Index.IIndexDetails indexDetails;
if (Index.IndexCategory.of(indexType) == Index.IndexCategory.ARRAY) {
if (!stmtCreateIndex.hasExcludeUnknownKey()
|| !stmtCreateIndex.getExcludeUnknownKey().getOrElse(false)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Array indexes must specify EXCLUDE UNKNOWN KEY.");
}
if (!hadUnnest) {
// prohibited by the grammar
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
String.valueOf(indexType));
}
if (stmtCreateIndex.isEnforced()) {
// not supported yet.
throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_INDEX_TYPE, sourceLoc,
String.valueOf(indexType));
}
List<Index.ArrayIndexElement> indexElementList = new ArrayList<>(indexedElementsCount);
for (int i = 0; i < indexedElementsCount; i++) {
CreateIndexStatement.IndexedElement indexedElement = indexedElements.get(i);
List<List<String>> projectList =
indexedElement.getProjectList().stream().map(Pair::getFirst).collect(Collectors.toList());
indexElementList.add(new Index.ArrayIndexElement(indexedElement.getUnnestList(), projectList,
indexFieldTypes.get(i), indexedElement.getSourceIndicator()));
}
indexDetails = new Index.ArrayIndexDetails(indexElementList, overridesFieldTypes);
} else {
List<List<String>> keyFieldNames = new ArrayList<>(indexedElementsCount);
List<IAType> keyFieldTypes = new ArrayList<>(indexedElementsCount);
List<Integer> keyFieldSourceIndicators = new ArrayList<>(indexedElementsCount);
// secondary primary indexes do not have search keys (no SKs), and thus no equivalent indicators
if (!isSecondaryPrimary) {
for (int i = 0; i < indexedElementsCount; i++) {
CreateIndexStatement.IndexedElement indexedElement = indexedElements.get(i);
keyFieldNames.add(indexedElement.getProjectList().get(0).first);
keyFieldTypes.add(indexFieldTypes.get(i).get(0));
keyFieldSourceIndicators.add(indexedElement.getSourceIndicator());
}
}
switch (Index.IndexCategory.of(indexType)) {
case VALUE:
Map<String, String> castConfig = TypeUtil.validateConfiguration(stmtCreateIndex.getCastConfig(),
stmtCreateIndex.getSourceLocation());
String datetimeFormat = TypeUtil.getDatetimeFormat(castConfig);
String dateFormat = TypeUtil.getDateFormat(castConfig);
String timeFormat = TypeUtil.getTimeFormat(castConfig);
indexDetails = new Index.ValueIndexDetails(keyFieldNames, keyFieldSourceIndicators,
keyFieldTypes, overridesFieldTypes, stmtCreateIndex.getExcludeUnknownKey(),
stmtCreateIndex.getCastDefaultNull(), datetimeFormat, dateFormat, timeFormat);
break;
case TEXT:
indexDetails = new Index.TextIndexDetails(keyFieldNames, keyFieldSourceIndicators,
keyFieldTypes, overridesFieldTypes, stmtCreateIndex.getGramLength(),
stmtCreateIndex.getFullTextConfigName());
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
String.valueOf(indexType));
}
}
Index newIndex = new Index(databaseName, dataverseName, datasetName, indexName, indexType, indexDetails,
stmtCreateIndex.isEnforced(), false, MetadataUtil.PENDING_ADD_OP, creator);
bActiveTxn = false; // doCreateIndexImpl() takes over the current transaction
EntityDetails entityDetails = EntityDetails.newIndex(databaseName, dataverseName, indexName);
doCreateIndexImpl(hcc, metadataProvider, ds, newIndex, jobFlags, sourceLoc, creator, entityDetails);
} catch (Exception e) {
if (bActiveTxn) {
abort(e, e, mdTxnCtx);
}
throw e;
}
}