in framework/entity/src/main/java/org/apache/ofbiz/entity/jdbc/DatabaseUtil.java [261:851]
public void checkDb(Map<String, ModelEntity> modelEntities, List<String> colWrongSize, List<String> messages, boolean checkPks,
boolean checkFks, boolean checkFkIdx, boolean addMissing) {
if (isLegacy) {
throw new RuntimeException("Cannot run checkDb on a legacy database connection; configure a database helper (entityengine.xml)");
}
ExecutorService executor = Executors.newFixedThreadPool(datasourceInfo.getMaxWorkerPoolSize());
UtilTimer timer = new UtilTimer();
timer.timerString("Start - Before Get Database Meta Data");
// get ALL tables from this database
TreeSet<String> tableNames = this.getTableNames(messages);
TreeSet<String> fkTableNames = tableNames == null ? null : new TreeSet<>(tableNames);
TreeSet<String> indexTableNames = tableNames == null ? null : new TreeSet<>(tableNames);
if (tableNames == null) {
String message = "Could not get table name information from the database, aborting.";
if (messages != null) messages.add(message);
Debug.logError(message, MODULE);
return;
}
timer.timerString("After Get All Table Names");
// get ALL column info, put into hashmap by table name
Map<String, Map<String, ColumnCheckInfo>> colInfo = getColumnInfo(tableNames, checkPks, messages, executor);
if (colInfo == null) {
String message = "Could not get column information from the database, aborting.";
if (messages != null) messages.add(message);
Debug.logError(message, MODULE);
return;
}
timer.timerString("After Get All Column Info");
// -make sure all entities have a corresponding table
// -list all tables that do not have a corresponding entity
// -display message if number of table columns does not match number of entity fields
// -list all columns that do not have a corresponding field
// -make sure each corresponding column is of the correct type
// -list all fields that do not have a corresponding column
timer.timerString("Before Individual Table/Column Check");
List<ModelEntity> modelEntityList = new ArrayList<>(modelEntities.values());
// sort using compareTo method on ModelEntity
modelEntityList.sort(null);
int curEnt = 0;
int totalEnt = modelEntityList.size();
List<ModelEntity> entitiesAdded = new LinkedList<>();
String schemaName;
try {
schemaName = getSchemaName(messages);
} catch (SQLException e) {
String message = "Could not get schema name the database, aborting.";
if (messages != null) messages.add(message);
Debug.logError(message, MODULE);
return;
}
List<Future<CreateTableCallable>> tableFutures = new LinkedList<>();
for (ModelEntity entity : modelEntityList) {
curEnt++;
// if this is a view entity, do not check it...
if (entity instanceof ModelViewEntity) {
String entMessage =
"(" + timer.timeSinceLast() + "ms) NOT Checking #" + curEnt + "/" + totalEnt + " View Entity " + entity.getEntityName();
Debug.logVerbose(entMessage, MODULE);
if (messages != null) messages.add(entMessage);
continue;
// if never-check is set then don't check it either
} else if (entity.getNeverCheck()) {
String entMessage =
"(" + timer.timeSinceLast() + "ms) NOT Checking #" + curEnt + "/" + totalEnt + " Entity " + entity.getEntityName();
Debug.logVerbose(entMessage, MODULE);
if (messages != null) messages.add(entMessage);
continue;
}
String plainTableName = entity.getPlainTableName();
String tableName;
if (UtilValidate.isNotEmpty(schemaName)) {
tableName = schemaName + "." + plainTableName;
} else {
tableName = plainTableName;
}
String entMessage = "(" + timer.timeSinceLast() + "ms) Checking #" + curEnt + "/" + totalEnt
+ " Entity " + entity.getEntityName() + " with table " + tableName;
Debug.logVerbose(entMessage, MODULE);
if (messages != null) messages.add(entMessage);
// -make sure all entities have a corresponding table
if (tableNames.contains(tableName)) {
tableNames.remove(tableName);
Map<String, ModelField> fieldColNames = new HashMap<>();
Iterator<ModelField> fieldIter = entity.getFieldsIterator();
while (fieldIter.hasNext()) {
ModelField field = fieldIter.next();
fieldColNames.put(field.getColName(), field);
}
Map<String, ColumnCheckInfo> colMap = colInfo.get(tableName);
if (colMap != null) {
for (ColumnCheckInfo ccInfo : colMap.values()) {
// -list all columns that do not have a corresponding field
if (fieldColNames.containsKey(ccInfo.columnName)) {
ModelField field = null;
field = fieldColNames.remove(ccInfo.columnName);
ModelFieldType modelFieldType = modelFieldTypeReader.getModelFieldType(field.getType());
if (modelFieldType != null) {
// make sure each corresponding column is of the correct type
String fullTypeStr = modelFieldType.getSqlType();
String typeName;
int columnSize = -1;
int decimalDigits = -1;
int openParen = fullTypeStr.indexOf('(');
int closeParen = fullTypeStr.indexOf(')');
int comma = fullTypeStr.indexOf(',');
if (openParen > 0 && closeParen > 0 && closeParen > openParen) {
typeName = fullTypeStr.substring(0, openParen);
if (!("DATETIME".equals(typeName) || "TIME".equals(typeName))) { // for DATETIME and TIME fields the number
// within the parenthesis doesn't represent the column size
if (comma > 0 && comma > openParen && comma < closeParen) {
String csStr = fullTypeStr.substring(openParen + 1, comma);
try {
columnSize = Integer.parseInt(csStr);
} catch (NumberFormatException e) {
Debug.logError(e, MODULE);
}
String ddStr = fullTypeStr.substring(comma + 1, closeParen);
try {
decimalDigits = Integer.parseInt(ddStr);
} catch (NumberFormatException e) {
Debug.logError(e, MODULE);
}
} else if (openParen + 1 < closeParen) {
String csStr = fullTypeStr.substring(openParen + 1, closeParen);
try {
columnSize = Integer.parseInt(csStr);
} catch (NumberFormatException e) {
Debug.logError(e, MODULE);
}
}
}
} else {
typeName = fullTypeStr;
}
// override the default typeName with the sqlTypeAlias if it is specified
if (UtilValidate.isNotEmpty(modelFieldType.getSqlTypeAlias())) {
typeName = modelFieldType.getSqlTypeAlias();
}
// NOTE: this may need a toUpperCase in some cases, keep an eye on it, okay just compare with ignore case
if (!ccInfo.typeName.equalsIgnoreCase(typeName)) {
String message = "Column [" + ccInfo.columnName + "] of table [" + tableName + "] of entity ["
+ entity.getEntityName() + "] is of type [" + ccInfo.typeName + "] in the database, but is "
+ "defined as type [" + typeName + "] in the entity definition.";
Debug.logError(message, MODULE);
if (messages != null) messages.add(message);
}
if (columnSize != -1 && ccInfo.columnSize != -1 && columnSize != ccInfo.columnSize && (columnSize * 3)
!= ccInfo.columnSize) {
String message = "Column [" + ccInfo.columnName + "] of table [" + tableName + "] of entity ["
+ entity.getEntityName() + "] has a column size of [" + ccInfo.columnSize
+ "] in the database, but is defined to have a column size of [" + columnSize + "] in the "
+ "entity definition.";
Debug.logWarning(message, MODULE);
if (messages != null) messages.add(message);
if (columnSize > ccInfo.columnSize && colWrongSize != null) {
// add item to list of wrong sized columns; only if the entity is larger
colWrongSize.add(entity.getEntityName() + "." + field.getName());
}
}
if (decimalDigits != -1 && decimalDigits != ccInfo.decimalDigits) {
String message = "Column [" + ccInfo.columnName + "] of table [" + tableName + "] of entity ["
+ entity.getEntityName() + "] has a decimalDigits of [" + ccInfo.decimalDigits
+ "] in the database, but is defined to have a decimalDigits of [" + decimalDigits + "] in the"
+ " entity definition.";
Debug.logWarning(message, MODULE);
if (messages != null) messages.add(message);
}
// do primary key matching check
if (checkPks && ccInfo.isPk && !field.getIsPk()) {
String message = "Column [" + ccInfo.columnName + "] of table [" + tableName + "] of entity ["
+ entity.getEntityName() + "] IS a primary key in the database, but IS NOT a primary key in "
+ "the entity definition. The primary key for this table needs to be re-created or modified so "
+ "that this column is NOT part of the primary key.";
Debug.logError(message, MODULE);
if (messages != null) messages.add(message);
}
if (checkPks && !ccInfo.isPk && field.getIsPk()) {
String message = "Column [" + ccInfo.columnName + "] of table [" + tableName + "] of entity ["
+ entity.getEntityName() + "] IS NOT a primary key in the database, but IS a primary key in "
+ "the entity definition. The primary key for this table needs to be re-created or modified to "
+ "add this column to the primary key. Note that data may need to be added first as a primary "
+ "key column cannot have an null values.";
Debug.logError(message, MODULE);
if (messages != null) messages.add(message);
}
} else {
String message =
"Column [" + ccInfo.columnName + "] of table [" + tableName + "] of entity [" + entity.getEntityName()
+ "] has a field type name of [" + field.getType() + "] which is not found in the field type definitions";
Debug.logError(message, MODULE);
if (messages != null) messages.add(message);
}
} else {
String message =
"Column [" + ccInfo.columnName + "] of table [" + tableName + "] of entity [" + entity.getEntityName() + "] "
+ "exists in the database but has no corresponding field" + ((checkPks && ccInfo.isPk)
? " (and it is a PRIMARY KEY COLUMN)" : "");
Debug.logWarning(message, MODULE);
if (messages != null) messages.add(message);
}
}
// -display message if number of table columns does not match number of entity fields
if (colMap.size() != entity.getFieldsSize()) {
String message =
"Entity [" + entity.getEntityName() + "] has " + entity.getFieldsSize() + " fields but table [" + tableName + "] "
+ "has " + colMap.size() + " columns.";
Debug.logWarning(message, MODULE);
if (messages != null) messages.add(message);
}
}
// -list all fields that do not have a corresponding column
for (ModelField field : fieldColNames.values()) {
String message = "Field [" + field.getName() + "] of entity [" + entity.getEntityName() + "] is missing its corresponding "
+ "column [" + field.getColName() + "]" + (field.getIsPk() ? " (and it is a PRIMARY KEY FIELD)" : "");
Debug.logWarning(message, MODULE);
if (messages != null) messages.add(message);
if (addMissing) {
// add the column
String errMsg = addColumn(entity, field);
if (UtilValidate.isNotEmpty(errMsg)) {
message = "Could not add column [" + field.getColName() + "] to table [" + tableName + "]: " + errMsg;
Debug.logError(message, MODULE);
if (messages != null) messages.add(message);
} else {
message = "Added column [" + field.getColName() + "] to table [" + tableName + "]" + (field.getIsPk()
? " (NOTE: this is a PRIMARY KEY FIELD, but the primary key was not updated automatically (not considered a"
+ "safe operation), be sure to fill in any needed data and re-create the primary key)" : "");
Debug.logImportant(message, MODULE);
if (messages != null) messages.add(message);
}
}
}
} else {
String message = "Entity [" + entity.getEntityName() + "] has no table in the database";
Debug.logWarning(message, MODULE);
if (messages != null) messages.add(message);
if (addMissing) {
// create the table
tableFutures.add(executor.submit(new CreateTableCallable(entity, modelEntities, tableName)));
}
}
}
for (CreateTableCallable tableCallable : ExecutionPool.getAllFutures(tableFutures)) {
tableCallable.updateData(messages, entitiesAdded);
}
timer.timerString("After Individual Table/Column Check");
// -list all tables that do not have a corresponding entity
for (String tableName : tableNames) {
String message = "Table named [" + tableName + "] exists in the database but has no corresponding entity";
Debug.logWarning(message, MODULE);
if (messages != null) messages.add(message);
}
// for each newly added table, add fk indices
if (datasourceInfo.getUseForeignKeyIndices()) {
int totalFkIndices = 0;
List<Future<AbstractCountingCallable>> fkIndicesFutures = new LinkedList<>();
for (ModelEntity curEntity : entitiesAdded) {
if (curEntity.getRelationsOneSize() > 0) {
fkIndicesFutures.add(executor.submit(new AbstractCountingCallable(curEntity, modelEntities) {
@Override
public AbstractCountingCallable call() throws Exception {
setCount(createForeignKeyIndices(getEntity(), datasourceInfo.getConstraintNameClipLength(), messages));
return this;
}
}));
}
}
for (AbstractCountingCallable fkIndicesCallable : ExecutionPool.getAllFutures(fkIndicesFutures)) {
totalFkIndices += fkIndicesCallable.updateData(messages);
}
if (totalFkIndices > 0) Debug.logImportant("==== TOTAL Foreign Key Indices Created: " + totalFkIndices, MODULE);
}
// for each newly added table, add fks
if (datasourceInfo.getUseForeignKeys()) {
int totalFks = 0;
for (ModelEntity curEntity : entitiesAdded) {
totalFks += this.createForeignKeys(curEntity, modelEntities, datasourceInfo.getConstraintNameClipLength(),
datasourceInfo.getFkStyle(), datasourceInfo.getUseFkInitiallyDeferred(), messages);
}
if (totalFks > 0) Debug.logImportant("==== TOTAL Foreign Keys Created: " + totalFks, MODULE);
}
// for each newly added table, add declared indexes
if (datasourceInfo.getUseIndices()) {
int totalDis = 0;
List<Future<AbstractCountingCallable>> disFutures = new LinkedList<>();
for (ModelEntity curEntity : entitiesAdded) {
if (curEntity.getIndexesSize() > 0) {
disFutures.add(executor.submit(new AbstractCountingCallable(curEntity, modelEntities) {
@Override
public AbstractCountingCallable call() throws Exception {
setCount(createDeclaredIndices(getEntity(), messages));
return this;
}
}));
}
}
for (AbstractCountingCallable disCallable : ExecutionPool.getAllFutures(disFutures)) {
totalDis += disCallable.updateData(messages);
}
if (totalDis > 0) Debug.logImportant("==== TOTAL Declared Indices Created: " + totalDis, MODULE);
}
// make sure each one-relation has an FK
if (checkFks) {
//if (!justColumns && datasourceInfo.getUseForeignKeys() && datasourceInfo.checkForeignKeysOnStart) {
// NOTE: This ISN'T working for Postgres or MySQL, who knows about others, may be from JDBC driver bugs...
int numFksCreated = 0;
// TODO: check each key-map to make sure it exists in the FK, if any differences warn and then remove FK and recreate it
// get ALL column info, put into hashmap by table name
Map<String, Map<String, ReferenceCheckInfo>> refTableInfoMap = this.getReferenceInfo(fkTableNames, messages);
// Debug.logVerbose("Ref Info Map: " + refTableInfoMap, MODULE);
if (refTableInfoMap == null) {
// uh oh, something happened while getting info...
if (Debug.verboseOn()) {
Debug.logVerbose("Ref Table Info Map is null", MODULE);
}
} else {
for (ModelEntity entity : modelEntityList) {
String entityName = entity.getEntityName();
// if this is a view entity, do not check it...
if (entity instanceof ModelViewEntity) {
String entMessage = "NOT Checking View Entity " + entity.getEntityName();
Debug.logVerbose(entMessage, MODULE);
if (messages != null) {
messages.add(entMessage);
}
continue;
}
// get existing FK map for this table
Map<String, ReferenceCheckInfo> rcInfoMap = refTableInfoMap.get(entity.getTableName(datasourceInfo));
// Debug.logVerbose("Got ref info for table " + entity.getTableName(datasourceInfo) + ": " + rcInfoMap, MODULE);
// go through each relation to see if an FK already exists
Iterator<ModelRelation> relations = entity.getRelationsIterator();
boolean createdConstraints = false;
while (relations.hasNext()) {
ModelRelation modelRelation = relations.next();
if (!"one".equals(modelRelation.getType())) {
continue;
}
ModelEntity relModelEntity = modelEntities.get(modelRelation.getRelEntityName());
if (relModelEntity == null) {
Debug.logError("No such relation: " + entity.getEntityName() + " -> " + modelRelation.getRelEntityName(), MODULE);
continue;
}
String relConstraintName = makeFkConstraintName(modelRelation, datasourceInfo.getConstraintNameClipLength());
ReferenceCheckInfo rcInfo = null;
if (rcInfoMap != null) {
rcInfo = rcInfoMap.get(relConstraintName);
}
if (rcInfo != null) {
rcInfoMap.remove(relConstraintName);
} else {
// if not, create one
String noFkMessage = "No Foreign Key Constraint [" + relConstraintName + "] found for entity [" + entityName + "]";
if (messages != null) messages.add(noFkMessage);
if (Debug.infoOn()) {
Debug.logInfo(noFkMessage, MODULE);
}
if (addMissing) {
String errMsg = createForeignKey(entity, modelRelation, relModelEntity,
datasourceInfo.getConstraintNameClipLength(), datasourceInfo.getFkStyle(),
datasourceInfo.getUseFkInitiallyDeferred());
if (UtilValidate.isNotEmpty(errMsg)) {
String message =
"Could not create foreign key " + relConstraintName + " for entity [" + entity.getEntityName() + "]: "
+ errMsg;
Debug.logError(message, MODULE);
if (messages != null) messages.add(message);
} else {
String message = "Created foreign key " + relConstraintName + " for entity [" + entity.getEntityName() + "]";
Debug.logVerbose(message, MODULE);
if (messages != null) messages.add(message);
createdConstraints = true;
numFksCreated++;
}
}
}
}
if (createdConstraints) {
String message = "Created foreign key(s) for entity [" + entity.getEntityName() + "]";
Debug.logImportant(message, MODULE);
if (messages != null) messages.add(message);
}
// show foreign key references that exist but are unknown
if (rcInfoMap != null) {
for (String rcKeyLeft : rcInfoMap.keySet()) {
String message = "Unknown Foreign Key Constraint " + rcKeyLeft + " found in table " + entity.getTableName(datasourceInfo);
Debug.logImportant(message, MODULE);
if (messages != null) messages.add(message);
}
}
}
}
if (Debug.infoOn()) {
Debug.logInfo("Created " + numFksCreated + " fk refs", MODULE);
}
}
// make sure each one-relation has an index
if (checkFkIdx || datasourceInfo.getCheckIndicesOnStart()) {
//if (!justColumns && datasourceInfo.getUseForeignKeyIndices() && datasourceInfo.checkFkIndicesOnStart) {
int numIndicesCreated = 0;
// TODO: check each key-map to make sure it exists in the index, if any differences warn and then remove the index and recreate it
// get ALL column info, put into hashmap by table name
boolean needsUpperCase[] = new boolean[1];
Map<String, Set<String>> tableIndexListMap = this.getIndexInfo(indexTableNames, messages, needsUpperCase);
// Debug.logVerbose("Ref Info Map: " + refTableInfoMap, MODULE);
if (tableIndexListMap == null) {
// uh oh, something happened while getting info...
if (Debug.verboseOn()) {
Debug.logVerbose("Ref Table Info Map is null", MODULE);
}
} else {
for (ModelEntity entity : modelEntityList) {
String entityName = entity.getEntityName();
// if this is a view entity, do not check it...
if (entity instanceof ModelViewEntity) {
String entMessage = "NOT Checking View Entity " + entity.getEntityName();
Debug.logVerbose(entMessage, MODULE);
if (messages != null) messages.add(entMessage);
continue;
}
// get existing index list for this table
Set<String> tableIndexList = tableIndexListMap.get(entity.getTableName(datasourceInfo));
// Debug.logVerbose("Got ind info for table " + entity.getTableName(datasourceInfo) + ": " + tableIndexList, MODULE);
if (tableIndexList == null) {
// evidently no indexes in the database for this table, do the create all
if (checkFkIdx) {
this.createForeignKeyIndices(entity, datasourceInfo.getConstraintNameClipLength(), messages);
}
if (datasourceInfo.getCheckIndicesOnStart()) {
this.createDeclaredIndices(entity, messages);
}
continue;
}
// go through each relation to see if an FK already exists
boolean createdConstraints = false;
Iterator<ModelRelation> relations = entity.getRelationsIterator();
while (relations.hasNext()) {
ModelRelation modelRelation = relations.next();
if (!"one".equals(modelRelation.getType())) {
continue;
}
String relConstraintName = makeFkConstraintName(modelRelation, datasourceInfo.getConstraintNameClipLength());
if (tableIndexList.contains(relConstraintName)) {
tableIndexList.remove(relConstraintName);
} else {
if (checkFkIdx) {
// if not, create one
String noIdxMessage = "No Index [" + relConstraintName + "] found for entity [" + entityName + "]";
if (messages != null) messages.add(noIdxMessage);
if (Debug.infoOn()) {
Debug.logInfo(noIdxMessage, MODULE);
}
if (addMissing) {
String errMsg = createForeignKeyIndex(entity, modelRelation, datasourceInfo.getConstraintNameClipLength());
if (UtilValidate.isNotEmpty(errMsg)) {
String message =
"Could not create foreign key index " + relConstraintName + " for entity [" + entity.getEntityName()
+ "]:"
+ " " + errMsg;
Debug.logError(message, MODULE);
if (messages != null) messages.add(message);
} else {
String message =
"Created foreign key index " + relConstraintName + " for entity [" + entity.getEntityName() + "]";
Debug.logVerbose(message, MODULE);
if (messages != null) messages.add(message);
createdConstraints = true;
numIndicesCreated++;
}
}
}
}
}
if (createdConstraints) {
String message = "Created foreign key index/indices for entity [" + entity.getEntityName() + "]";
Debug.logImportant(message, MODULE);
if (messages != null) messages.add(message);
}
// go through each indice to see if an indice already exists
boolean createdIndexes = false;
Iterator<ModelIndex> indexes = entity.getIndexesIterator();
while (indexes.hasNext()) {
ModelIndex modelIndex = indexes.next();
String relIndexName = makeIndexName(modelIndex, datasourceInfo.getConstraintNameClipLength());
String checkIndexName = needsUpperCase[0] ? relIndexName.toUpperCase() : relIndexName;
if (tableIndexList.contains(checkIndexName)) {
tableIndexList.remove(checkIndexName);
} else {
if (datasourceInfo.getCheckIndicesOnStart()) {
// if not, create one
String noIdxMessage = "No Index [" + relIndexName + "] found for entity [" + entityName + "]";
if (messages != null) messages.add(noIdxMessage);
if (Debug.infoOn()) {
Debug.logInfo(noIdxMessage, MODULE);
}
if (addMissing) {
String errMsg = createDeclaredIndex(entity, modelIndex);
if (UtilValidate.isNotEmpty(errMsg)) {
String message = "Could not create index " + relIndexName + " for entity [" + entity.getEntityName() + "]: "
+ errMsg;
Debug.logError(message, MODULE);
if (messages != null) messages.add(message);
} else {
String message = "Created index " + relIndexName + " for entity [" + entity.getEntityName() + "]";
Debug.logVerbose(message, MODULE);
if (messages != null) messages.add(message);
createdIndexes = true;
numIndicesCreated++;
}
}
}
}
}
if (createdIndexes) {
String message = "Created declared index/indices for entity [" + entity.getEntityName() + "]";
Debug.logImportant(message, MODULE);
if (messages != null) messages.add(message);
}
// show index key references that exist but are unknown
for (String indexLeft : tableIndexList) {
String message = "Unknown Index " + indexLeft + " found in table " + entity.getTableName(datasourceInfo);
Debug.logImportant(message, MODULE);
if (messages != null) messages.add(message);
}
}
}
if (numIndicesCreated > 0 && Debug.infoOn()) Debug.logInfo("Created " + numIndicesCreated + " indices", MODULE);
}
executor.shutdown();
timer.timerString("Finished Checking Entity Database");
}