in entity-store/src/main/kotlin/jetbrains/exodus/entitystore/PersistentEntityStoreRefactorings.kt [577:764]
fun refactorMakePropTablesConsistent() {
store.executeInReadonlyTransaction { txn ->
txn as PersistentStoreTransaction
for (entityType in store.getEntityTypes(txn)) {
logInfo("Refactoring making props' tables consistent for [$entityType]")
runReadonlyTransactionSafeForEntityType(entityType) {
val entityTypeId = store.getEntityTypeId(txn, entityType, false)
val propTable = store.getPropertiesTable(txn, entityTypeId)
val envTxn = txn.environmentTransaction
val props = IntHashMap<LongHashMap<PropertyValue>>()
val all = (txn.getAll(entityType) as EntityIterableBase).toSet(txn)
val propertyTypes = store.propertyTypes
val entitiesToDelete = LongHashSet()
store.getPrimaryPropertyIndexCursor(txn, propTable).use { cursor ->
while (cursor.next) {
val propKey = PropertyKey.entryToPropertyKey(cursor.key)
val localId = propKey.entityLocalId
if (!all.contains(entityTypeId, localId)) {
entitiesToDelete.add(localId)
continue
}
val propValue = propertyTypes.entryToPropertyValue(cursor.value)
val propId = propKey.propertyId
var entitiesToValues = props[propId]
if (entitiesToValues == null) {
entitiesToValues = LongHashMap()
props[propId] = entitiesToValues
}
entitiesToValues[localId] = propValue
}
}
if (!entitiesToDelete.isEmpty()) {
store.executeInExclusiveTransaction { txn ->
txn as PersistentStoreTransaction
for (localId in entitiesToDelete) {
store.deleteEntity(
txn,
PersistentEntity(store, PersistentEntityId(entityTypeId, localId))
)
}
}
}
val missingPairs = ArrayList<Pair<Int, Pair<ByteIterable, ByteIterable>>>()
val allPropsMap = IntHashMap<MutableSet<Long>>()
for (propId in props.keys) {
val valueIndex = propTable.getValueIndex(txn, propId, false)
val valueCursor = valueIndex?.openCursor(envTxn)
val entitiesToValues = props[propId]
entitiesToValues?.let {
val localIdSet: Set<Long> = entitiesToValues.keys
val sortedLocalIdSet = TreeSet(localIdSet)
allPropsMap[propId] = sortedLocalIdSet
val localIds = sortedLocalIdSet.toTypedArray()
for (localId in localIds) {
val propValue = checkNotNull(entitiesToValues[localId])
for (secondaryKey in PropertiesTable.createSecondaryKeys(
propertyTypes, PropertyTypes.propertyValueToEntry(propValue), propValue.type
)) {
val secondaryValue: ByteIterable = LongBinding.longToCompressedEntry(localId)
if (valueCursor == null || !valueCursor.getSearchBoth(
secondaryKey,
secondaryValue
)
) {
missingPairs.add(propId to (secondaryKey to secondaryValue))
}
}
}
}
valueCursor?.close()
}
if (missingPairs.isNotEmpty()) {
store.executeInExclusiveTransaction { tx ->
val txn = tx as PersistentStoreTransaction
for (pair in missingPairs) {
val valueIndex = propTable.getValueIndex(txn, pair.first, true)
val missing = pair.second
if (valueIndex == null) {
throw NullPointerException("Can't be")
}
valueIndex.put(txn.environmentTransaction, missing.first, missing.second)
}
}
logInfo("${missingPairs.size} missing secondary keys found and fixed for [$entityType]")
}
val phantomPairs = ArrayList<Pair<Int, Pair<ByteIterable, ByteIterable>>>()
for ((propId, value1) in propTable.valueIndices) {
val entitiesToValues = props[propId] ?: continue
val c = value1.openCursor(envTxn)
while (c.next) {
val keyEntry = c.key
val valueEntry = c.value
val propValue = entitiesToValues[LongBinding.compressedEntryToLong(valueEntry)]
if (propValue != null) {
val data = propValue.data
val typeId = propValue.type.typeId
val dataClass: Class<out Comparable<Any>>?
val objectBinding: ComparableBinding
if (typeId == ComparableValueType.COMPARABLE_SET_VALUE_TYPE) {
@Suppress("UNCHECKED_CAST")
dataClass = (data as ComparableSet<Comparable<Any>>).itemClass
if (dataClass == null) {
phantomPairs.add(
propId to (ArrayByteIterable(keyEntry) to ArrayByteIterable(
valueEntry
))
)
continue
}
objectBinding = propertyTypes.getPropertyType(dataClass).binding
} else {
dataClass = data.javaClass
objectBinding = propValue.binding
}
try {
val value = objectBinding.entryToObject(keyEntry)
if (dataClass == value.javaClass) {
if (typeId == ComparableValueType.COMPARABLE_SET_VALUE_TYPE) {
if ((data as ComparableSet<*>).containsItem(value)) {
continue
}
} else if (PropertyTypes.toLowerCase(data).compareTo(value) == 0) {
continue
}
}
} catch (t: Throwable) {
logger.error("Error reading property value index ", t)
throwJVMError(t)
}
}
phantomPairs.add(propId to (ArrayByteIterable(keyEntry) to ArrayByteIterable(valueEntry)))
}
c.close()
}
if (phantomPairs.isNotEmpty()) {
store.executeInExclusiveTransaction { tx ->
val txn = tx as PersistentStoreTransaction
val envTxn = txn.environmentTransaction
for (pair in phantomPairs) {
val valueIndex = propTable.getValueIndex(txn, pair.first, true)
val phantom = pair.second
if (valueIndex == null) {
throw NullPointerException("Can't be")
}
deletePair(valueIndex.openCursor(envTxn), phantom.first, phantom.second)
}
}
logInfo("${phantomPairs.size} phantom secondary keys found and fixed for [$entityType]")
}
val phantomIds = ArrayList<Pair<Int, Long>>()
propTable.allPropsIndex.iterable(envTxn, 0).forEach { pair ->
val propId = pair.first
val localId = pair.second
val localIds = allPropsMap[propId]
if (localIds == null || !localIds.remove(localId)) {
phantomIds.add(Pair(propId, localId))
} else if (localIds.isEmpty()) {
allPropsMap.remove(propId)
}
}
if (!allPropsMap.isEmpty()) {
val added = store.computeInExclusiveTransaction { txn ->
var count = 0
val allPropsIndex = propTable.allPropsIndex
val envTxn = (txn as PersistentStoreTransaction).environmentTransaction
for ((key, value) in allPropsMap) {
for (localId in value) {
allPropsIndex.put(envTxn, key, localId)
++count
}
}
count
}
logInfo("$added missing id pairs found and fixed for [$entityType]")
}
if (phantomIds.isNotEmpty()) {
store.executeInExclusiveTransaction { txn ->
val envTxn = (txn as PersistentStoreTransaction).environmentTransaction
phantomIds.forEach { phantom ->
propTable.allPropsIndex.remove(envTxn, phantom.first, phantom.second)
}
}
logInfo("${phantomIds.size} phantom id pairs found and fixed for [$entityType]")
}
}
}
}
}