in asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java [153:552]
protected Index createMetadataEntityFromARecord(ARecord indexRecord) throws AlgebricksException {
String dataverseCanonicalName =
((AString) indexRecord.getValueByPos(indexEntity.dataverseNameIndex())).getStringValue();
DataverseName dataverseName = DataverseName.createFromCanonicalForm(dataverseCanonicalName);
int databaseNameIndex = indexEntity.databaseNameIndex();
String databaseName;
if (databaseNameIndex >= 0) {
databaseName = ((AString) indexRecord.getValueByPos(databaseNameIndex)).getStringValue();
} else {
databaseName = MetadataUtil.databaseFor(dataverseName);
}
String datasetName = ((AString) indexRecord.getValueByPos(indexEntity.datasetNameIndex())).getStringValue();
String indexName = ((AString) indexRecord.getValueByPos(indexEntity.indexNameIndex())).getStringValue();
IndexType indexType = IndexType
.valueOf(((AString) indexRecord.getValueByPos(indexEntity.indexStructureIndex())).getStringValue());
boolean isPrimaryIndex = ((ABoolean) indexRecord.getValueByPos(indexEntity.isPrimaryIndex())).getBoolean();
// Read key names
List<Pair<List<List<String>>, List<List<String>>>> searchElements = new ArrayList<>();
switch (Index.IndexCategory.of(indexType)) {
case VALUE:
case TEXT:
// Read the key names from the SearchKeyName field
IACursor fieldNameCursor =
((AOrderedList) indexRecord.getValueByPos(indexEntity.searchKeyIndex())).getCursor();
AOrderedList fieldNameList;
while (fieldNameCursor.next()) {
fieldNameList = (AOrderedList) fieldNameCursor.get();
IACursor nestedFieldNameCursor = (fieldNameList.getCursor());
List<String> nestedFieldName = new ArrayList<>();
while (nestedFieldNameCursor.next()) {
nestedFieldName.add(((AString) nestedFieldNameCursor.get()).getStringValue());
}
searchElements.add(new Pair<>(null, Collections.singletonList(nestedFieldName)));
}
break;
case ARRAY:
// Read the unnest/project from the ComplexSearchKeyName field
int complexSearchKeyFieldPos = indexRecord.getType().getFieldIndex(INDEX_SEARCHKEY_ELEMENTS_FIELD_NAME);
IACursor complexSearchKeyCursor = new ACollectionCursor();
if (complexSearchKeyFieldPos > 0) {
complexSearchKeyCursor =
((AOrderedList) indexRecord.getValueByPos(complexSearchKeyFieldPos)).getCursor();
}
while (complexSearchKeyCursor.next()) {
Pair<List<List<String>>, List<List<String>>> searchElement;
IAObject complexSearchKeyItem = complexSearchKeyCursor.get();
switch (complexSearchKeyItem.getType().getTypeTag()) {
case ARRAY:
AOrderedList complexSearchKeyArray = (AOrderedList) complexSearchKeyItem;
List<String> project = new ArrayList<>(complexSearchKeyArray.size());
// We only have one element.
AOrderedList innerListForArray = (AOrderedList) complexSearchKeyArray.getItem(0);
IACursor innerListCursorForArray = innerListForArray.getCursor();
while (innerListCursorForArray.next()) {
project.add(((AString) innerListCursorForArray.get()).getStringValue());
}
searchElement = new Pair<>(null, Collections.singletonList(project));
break;
case OBJECT:
ARecord complexSearchKeyRecord = (ARecord) complexSearchKeyItem;
ARecordType complexSearchKeyRecordType = complexSearchKeyRecord.getType();
int unnestFieldPos =
complexSearchKeyRecordType.getFieldIndex(COMPLEXSEARCHKEY_UNNEST_FIELD_NAME);
if (unnestFieldPos < 0) {
throw new AsterixException(ErrorCode.METADATA_ERROR, complexSearchKeyRecord.toJSON());
}
AOrderedList unnestFieldList =
(AOrderedList) complexSearchKeyRecord.getValueByPos(unnestFieldPos);
List<List<String>> unnestList = new ArrayList<>(unnestFieldList.size());
IACursor unnestFieldListCursor = unnestFieldList.getCursor();
while (unnestFieldListCursor.next()) {
AOrderedList innerList = (AOrderedList) unnestFieldListCursor.get();
List<String> unnestPath = new ArrayList<>(innerList.size());
IACursor innerListCursor = innerList.getCursor();
while (innerListCursor.next()) {
unnestPath.add(((AString) innerListCursor.get()).getStringValue());
}
unnestList.add(unnestPath);
}
int projectFieldPos =
complexSearchKeyRecordType.getFieldIndex(COMPLEXSEARCHKEY_PROJECT_FIELD_NAME);
List<List<String>> projectList = new ArrayList<>();
if (projectFieldPos >= 0) {
AOrderedList projectFieldList =
(AOrderedList) complexSearchKeyRecord.getValueByPos(projectFieldPos);
projectList = new ArrayList<>(projectFieldList.size());
IACursor projectFieldListCursor = projectFieldList.getCursor();
while (projectFieldListCursor.next()) {
AOrderedList innerList = (AOrderedList) projectFieldListCursor.get();
List<String> projectPath = new ArrayList<>(innerList.size());
IACursor innerListCursor = innerList.getCursor();
while (innerListCursor.next()) {
projectPath.add(((AString) innerListCursor.get()).getStringValue());
}
projectList.add(projectPath);
}
} else {
projectList.add(null);
}
searchElement = new Pair<>(unnestList, projectList);
break;
default:
throw new AsterixException(ErrorCode.METADATA_ERROR, complexSearchKeyItem.toJSON());
}
searchElements.add(searchElement);
}
break;
case SAMPLE:
searchElements = Collections.emptyList();
break;
default:
throw new AsterixException(ErrorCode.METADATA_ERROR, indexType.toString());
}
int searchElementCount = searchElements.size();
String fullTextConfig = null;
int fullTextConfigPos = indexRecord.getType().getFieldIndex(FULL_TEXT_CONFIG_FIELD_NAME);
if (fullTextConfigPos >= 0) {
fullTextConfig = ((AString) indexRecord.getValueByPos(fullTextConfigPos)).getStringValue();
}
// Read a field-source-indicator field.
List<Integer> keyFieldSourceIndicator = new ArrayList<>(searchElementCount);
int keyFieldSourceIndicatorIndex =
indexRecord.getType().getFieldIndex(INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME);
if (keyFieldSourceIndicatorIndex >= 0) {
IACursor cursor = ((AOrderedList) indexRecord.getValueByPos(keyFieldSourceIndicatorIndex)).getCursor();
while (cursor.next()) {
keyFieldSourceIndicator.add((int) ((AInt8) cursor.get()).getByteValue());
}
} else {
for (int index = 0; index < searchElementCount; ++index) {
keyFieldSourceIndicator.add(Index.RECORD_INDICATOR);
}
}
// Read key types
int indexKeyTypeFieldPos = indexRecord.getType().getFieldIndex(INDEX_SEARCHKEY_TYPE_FIELD_NAME);
IACursor fieldTypeCursor = new ACollectionCursor();
if (indexKeyTypeFieldPos > 0) {
fieldTypeCursor = ((AOrderedList) indexRecord.getValueByPos(indexKeyTypeFieldPos)).getCursor();
}
List<List<IAType>> searchKeyType = new ArrayList<>(searchElementCount);
while (fieldTypeCursor.next()) {
IAObject fieldTypeItem = fieldTypeCursor.get();
switch (fieldTypeItem.getType().getTypeTag()) {
case STRING:
// This is a simple element, place in a single-element list.
String typeName = ((AString) fieldTypeItem).getStringValue();
IAType fieldType =
Datatype.getTypeFromTypeName(metadataNode, txnId, databaseName, dataverseName, typeName);
searchKeyType.add(Collections.singletonList(fieldType));
break;
case ARRAY:
// This is a complex element, read all types.
List<IAType> fieldTypes = new ArrayList<>();
AOrderedList fieldTypeList = (AOrderedList) fieldTypeItem;
IACursor fieldTypeListCursor = fieldTypeList.getCursor();
while (fieldTypeListCursor.next()) {
typeName = ((AString) fieldTypeListCursor.get()).getStringValue();
fieldTypes.add(Datatype.getTypeFromTypeName(metadataNode, txnId, databaseName, dataverseName,
typeName));
}
searchKeyType.add(fieldTypes);
break;
default:
throw new AsterixException(ErrorCode.METADATA_ERROR, fieldTypeItem.toJSON());
}
}
boolean isOverridingKeyTypes;
if (searchKeyType.isEmpty()) {
// if index key type information is not persisted, then we extract type information
// from the record metadata
Dataset dataset = metadataNode.getDataset(txnId, databaseName, dataverseName, datasetName);
String datatypeName = dataset.getItemTypeName();
DataverseName datatypeDataverseName = dataset.getItemTypeDataverseName();
String datatypeDatabase = dataset.getItemTypeDatabaseName();
ARecordType recordDt = (ARecordType) metadataNode
.getDatatype(txnId, datatypeDatabase, datatypeDataverseName, datatypeName).getDatatype();
String metatypeName = dataset.getMetaItemTypeName();
DataverseName metatypeDataverseName = dataset.getMetaItemTypeDataverseName();
String metaTypeDatabase = dataset.getMetaItemTypeDatabaseName();
ARecordType metaDt = null;
if (metatypeName != null && metatypeDataverseName != null) {
metaDt = (ARecordType) metadataNode
.getDatatype(txnId, metaTypeDatabase, metatypeDataverseName, metatypeName).getDatatype();
}
recordDt = (ARecordType) MetadataManagerUtil.findTypeForDatasetWithoutType(recordDt, metaDt, dataset);
searchKeyType = new ArrayList<>(searchElementCount);
for (int i = 0; i < searchElementCount; i++) {
Pair<List<List<String>>, List<List<String>>> searchElement = searchElements.get(i);
List<List<String>> unnestPathList = searchElement.first;
List<List<String>> projectPathList = searchElement.second;
ARecordType sourceRecordType = keyFieldSourceIndicator.get(i) == 1 ? metaDt : recordDt;
IAType inputTypePrime;
boolean inputTypeNullable, inputTypeMissable;
if (unnestPathList == null) {
inputTypePrime = sourceRecordType;
inputTypeNullable = inputTypeMissable = false;
} else {
Triple<IAType, Boolean, Boolean> unnestTypeResult =
KeyFieldTypeUtil.getKeyUnnestType(sourceRecordType, unnestPathList, null);
if (unnestTypeResult == null) {
inputTypePrime = null; // = ANY
inputTypeNullable = inputTypeMissable = true;
} else {
inputTypePrime = unnestTypeResult.first;
inputTypeNullable = unnestTypeResult.second;
inputTypeMissable = unnestTypeResult.third;
}
}
List<IAType> projectTypeList = new ArrayList<>(projectPathList.size());
for (List<String> projectPath : projectPathList) {
IAType projectTypePrime;
boolean projectTypeNullable, projectTypeMissable;
if (projectPath == null) {
projectTypePrime = inputTypePrime;
projectTypeNullable = inputTypeNullable;
projectTypeMissable = inputTypeMissable;
} else if (inputTypePrime == null ||
// handle special case of the empty field name in
// ExternalIndexingOperations.FILE_INDEX_FIELD_NAMES
(projectPath.size() == 1 && projectPath.get(0).isEmpty())) {
projectTypePrime = null; // ANY
projectTypeNullable = projectTypeMissable = true;
} else {
if (inputTypePrime.getTypeTag() != ATypeTag.OBJECT) {
throw new AsterixException(ErrorCode.METADATA_ERROR, projectPath.toString());
}
Triple<IAType, Boolean, Boolean> projectTypeResult =
KeyFieldTypeUtil.getKeyProjectType((ARecordType) inputTypePrime, projectPath, null);
if (projectTypeResult == null) {
if (indexType != IndexType.BTREE) {
throw new AsterixException(ErrorCode.METADATA_ERROR, projectPath.toString());
}
projectTypePrime = BuiltinType.ANY;
// We do not want the type to be union of Any, null and missing. Any will cover it all.
projectTypeNullable = false;
projectTypeMissable = false;
} else {
projectTypePrime = projectTypeResult.first;
projectTypeNullable = inputTypeNullable || projectTypeResult.second;
projectTypeMissable = inputTypeMissable || projectTypeResult.third;
}
}
IAType projectType = projectTypePrime == null ? null
: KeyFieldTypeUtil.makeUnknownableType(projectTypePrime, projectTypeNullable,
projectTypeMissable);
projectTypeList.add(projectType);
}
searchKeyType.add(projectTypeList);
}
isOverridingKeyTypes = false;
} else {
isOverridingKeyTypes = true;
}
// create index details structure
Index.IIndexDetails indexDetails;
switch (Index.IndexCategory.of(indexType)) {
case VALUE:
List<List<String>> keyFieldNames =
searchElements.stream().map(Pair::getSecond).map(l -> l.get(0)).collect(Collectors.toList());
List<IAType> keyFieldTypes = searchKeyType.stream().map(l -> l.get(0)).collect(Collectors.toList());
OptionalBoolean excludeUnknownKey = OptionalBoolean.empty();
OptionalBoolean castDefaultNull = OptionalBoolean.empty();
String datetimeFormat = null, dateFormat = null, timeFormat = null;
boolean isBtreeIdx = indexType == IndexType.BTREE && !isPrimaryIndex && !keyFieldNames.isEmpty();
if (isBtreeIdx) {
// exclude unknown key value; default to always include unknowns for normal b-trees
excludeUnknownKey = OptionalBoolean.FALSE();
int excludeUnknownKeyPos = indexRecord.getType().getFieldIndex(INDEX_EXCLUDE_UNKNOWN_FIELD_NAME);
if (excludeUnknownKeyPos >= 0) {
excludeUnknownKey = OptionalBoolean
.of(((ABoolean) indexRecord.getValueByPos(excludeUnknownKeyPos)).getBoolean());
}
// cast record
int castPos = indexRecord.getType().getFieldIndex(FIELD_NAME_CAST);
if (castPos >= 0) {
IAObject recValue = indexRecord.getValueByPos(castPos);
if (recValue.getType().getTypeTag() == ATypeTag.OBJECT) {
ARecord castRec = (ARecord) recValue;
ARecordType castRecType = castRec.getType();
// cast default value
int defaultFieldPos = castRecType.getFieldIndex(FIELD_NAME_DEFAULT);
if (defaultFieldPos >= 0) {
IAObject defaultVal = castRec.getValueByPos(defaultFieldPos);
if (defaultVal.getType().getTypeTag() == ATypeTag.NULL) {
castDefaultNull = OptionalBoolean.TRUE();
// Format fields
Triple<String, String, String> dateTimeFormats = getDateTimeFormats(castRec);
datetimeFormat = dateTimeFormats.first;
dateFormat = dateTimeFormats.second;
timeFormat = dateTimeFormats.third;
}
}
}
}
}
indexDetails = new Index.ValueIndexDetails(keyFieldNames, keyFieldSourceIndicator, keyFieldTypes,
isOverridingKeyTypes, excludeUnknownKey, castDefaultNull, datetimeFormat, dateFormat,
timeFormat);
break;
case TEXT:
keyFieldNames =
searchElements.stream().map(Pair::getSecond).map(l -> l.get(0)).collect(Collectors.toList());
keyFieldTypes = searchKeyType.stream().map(l -> l.get(0)).collect(Collectors.toList());
// Check if there is a gram length as well.
int gramLength = -1;
int gramLenPos = indexRecord.getType().getFieldIndex(GRAM_LENGTH_FIELD_NAME);
if (gramLenPos >= 0) {
gramLength = ((AInt32) indexRecord.getValueByPos(gramLenPos)).getIntegerValue();
}
indexDetails = new Index.TextIndexDetails(keyFieldNames, keyFieldSourceIndicator, keyFieldTypes,
isOverridingKeyTypes, gramLength, fullTextConfig);
break;
case ARRAY:
List<Index.ArrayIndexElement> elementList = new ArrayList<>(searchElementCount);
for (int i = 0; i < searchElementCount; i++) {
Pair<List<List<String>>, List<List<String>>> searchElement = searchElements.get(i);
List<IAType> typeList = searchKeyType.get(i);
int sourceIndicator = keyFieldSourceIndicator.get(i);
elementList.add(new Index.ArrayIndexElement(searchElement.first, searchElement.second, typeList,
sourceIndicator));
}
indexDetails = new Index.ArrayIndexDetails(elementList, isOverridingKeyTypes);
break;
case SAMPLE:
keyFieldNames =
searchElements.stream().map(Pair::getSecond).map(l -> l.get(0)).collect(Collectors.toList());
keyFieldTypes = searchKeyType.stream().map(l -> l.get(0)).collect(Collectors.toList());
int sampleSeedPos = indexRecord.getType().getFieldIndex(SAMPLE_SEED);
if (sampleSeedPos < 0) {
throw new AsterixException(ErrorCode.METADATA_ERROR, SAMPLE_SEED);
}
long sampleSeed = ((AInt64) indexRecord.getValueByPos(sampleSeedPos)).getLongValue();
int sampleCardinalityTargetPos = indexRecord.getType().getFieldIndex(SAMPLE_CARDINALITY_TARGET);
if (sampleCardinalityTargetPos < 0) {
throw new AsterixException(ErrorCode.METADATA_ERROR, SAMPLE_CARDINALITY_TARGET);
}
int sampleCardinalityTarget =
((AInt32) indexRecord.getValueByPos(sampleCardinalityTargetPos)).getIntegerValue();
int sourceCardinalityPos = indexRecord.getType().getFieldIndex(SOURCE_CARDINALITY);
if (sourceCardinalityPos < 0) {
throw new AsterixException(ErrorCode.METADATA_ERROR, SOURCE_CARDINALITY);
}
long sourceCardinality = ((AInt64) indexRecord.getValueByPos(sourceCardinalityPos)).getLongValue();
int sourceAvgItemSizePos = indexRecord.getType().getFieldIndex(SOURCE_AVG_ITEM_SIZE);
if (sourceAvgItemSizePos < 0) {
throw new AsterixException(ErrorCode.METADATA_ERROR, SOURCE_AVG_ITEM_SIZE);
}
int sourceAvgItemSize = ((AInt32) indexRecord.getValueByPos(sourceAvgItemSizePos)).getIntegerValue();
int indexesStatsPos = indexRecord.getType().getFieldIndex(INDEXES_STATS);
Map<String, IndexStats> indexesStats;
if (indexesStatsPos >= 0) {
AOrderedList indexesStatsList = (AOrderedList) indexRecord.getValueByPos(indexesStatsPos);
int numIndexes = indexesStatsList.size();
indexesStats = numIndexes > 0 ? new HashMap<>() : Collections.emptyMap();
for (int i = 0; i < numIndexes; i++) {
ARecord stats = (ARecord) indexesStatsList.getItem(i);
IAObject numPages = stats.getValueByPos(stats.getType().getFieldIndex(STATS_NUM_PAGES));
IAObject idxNameObj = stats.getValueByPos(stats.getType().getFieldIndex(STATS_INDEX_NAME));
String idxName = ((AString) idxNameObj).getStringValue();
IndexStats idxStats = new IndexStats(idxName, ((AInt64) numPages).getLongValue());
indexesStats.put(idxName, idxStats);
}
} else {
indexesStats = Collections.emptyMap();
}
indexDetails = new Index.SampleIndexDetails(keyFieldNames, keyFieldSourceIndicator, keyFieldTypes,
sampleCardinalityTarget, sourceCardinality, sourceAvgItemSize, sampleSeed, indexesStats);
break;
default:
throw new AsterixException(ErrorCode.METADATA_ERROR, indexType.toString());
}
int isEnforcedFieldPos = indexRecord.getType().getFieldIndex(INDEX_ISENFORCED_FIELD_NAME);
Boolean isEnforcingKeys = false;
if (isEnforcedFieldPos > 0) {
isEnforcingKeys = ((ABoolean) indexRecord.getValueByPos(isEnforcedFieldPos)).getBoolean();
}
int pendingOp = ((AInt32) indexRecord.getValueByPos(indexEntity.pendingOpIndex())).getIntegerValue();
Creator creator = Creator.createOrDefault(indexRecord);
return new Index(databaseName, dataverseName, datasetName, indexName, indexType, indexDetails, isEnforcingKeys,
isPrimaryIndex, pendingOp, creator);
}