in modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/AiravataDataMigrator.java [68:427]
public static void main(String[] args) throws SQLException, ClassNotFoundException, TException, ApplicationSettingsException {
String gatewayId = null;
if (args.length > 0) {
gatewayId = args[0];
}
String gatewayWhereClause = "";
if (gatewayId != null) {
System.out.println("Running sharing data migration for " + gatewayId);
gatewayWhereClause = " WHERE GATEWAY_ID = '" + gatewayId + "'";
} else {
System.out.println("Running sharing data migration for all gateways");
}
Connection expCatConnection = ConnectionFactory.getInstance().getExpCatConnection();
SharingRegistryServerHandler sharingRegistryServerHandler = new SharingRegistryServerHandler();
CredentialStoreService.Client credentialStoreServiceClient = getCredentialStoreServiceClient();
IamAdminServices.Client iamAdminServiceClient = getIamAdminServiceClient();
String query = "SELECT * FROM GATEWAY" + gatewayWhereClause;
Statement statement = expCatConnection.createStatement();
ResultSet rs = statement.executeQuery(query);
while (rs.next()) {
try{
//Creating domain entries
Domain domain = new Domain();
domain.setDomainId(rs.getString("GATEWAY_ID"));
domain.setName(rs.getString("GATEWAY_ID"));
domain.setDescription("Domain entry for " + domain.getName());
if (!sharingRegistryServerHandler.isDomainExists(domain.getDomainId()))
sharingRegistryServerHandler.createDomain(domain);
//Creating Entity Types for each domain
EntityType entityType = new EntityType();
entityType.setEntityTypeId(domain.getDomainId()+":PROJECT");
entityType.setDomainId(domain.getDomainId());
entityType.setName("PROJECT");
entityType.setDescription("Project entity type");
if (!sharingRegistryServerHandler.isEntityTypeExists(entityType.getDomainId(), entityType.getEntityTypeId()))
sharingRegistryServerHandler.createEntityType(entityType);
entityType = new EntityType();
entityType.setEntityTypeId(domain.getDomainId()+":EXPERIMENT");
entityType.setDomainId(domain.getDomainId());
entityType.setName("EXPERIMENT");
entityType.setDescription("Experiment entity type");
if (!sharingRegistryServerHandler.isEntityTypeExists(entityType.getDomainId(), entityType.getEntityTypeId()))
sharingRegistryServerHandler.createEntityType(entityType);
entityType = new EntityType();
entityType.setEntityTypeId(domain.getDomainId()+":FILE");
entityType.setDomainId(domain.getDomainId());
entityType.setName("FILE");
entityType.setDescription("File entity type");
if (!sharingRegistryServerHandler.isEntityTypeExists(entityType.getDomainId(), entityType.getEntityTypeId()))
sharingRegistryServerHandler.createEntityType(entityType);
entityType = new EntityType();
entityType.setEntityTypeId(domain.getDomainId()+":"+ ResourceType.APPLICATION_DEPLOYMENT.name());
entityType.setDomainId(domain.getDomainId());
entityType.setName("APPLICATION-DEPLOYMENT");
entityType.setDescription("Application Deployment entity type");
if (!sharingRegistryServerHandler.isEntityTypeExists(entityType.getDomainId(), entityType.getEntityTypeId()))
sharingRegistryServerHandler.createEntityType(entityType);
entityType = new EntityType();
entityType.setEntityTypeId(domain.getDomainId()+":"+ResourceType.GROUP_RESOURCE_PROFILE.name());
entityType.setDomainId(domain.getDomainId());
entityType.setName(ResourceType.GROUP_RESOURCE_PROFILE.name());
entityType.setDescription("Group Resource Profile entity type");
if (!sharingRegistryServerHandler.isEntityTypeExists(entityType.getDomainId(), entityType.getEntityTypeId()))
sharingRegistryServerHandler.createEntityType(entityType);
entityType = new EntityType();
entityType.setEntityTypeId(domain.getDomainId()+":"+ResourceType.CREDENTIAL_TOKEN.name());
entityType.setDomainId(domain.getDomainId());
entityType.setName(ResourceType.CREDENTIAL_TOKEN.name());
entityType.setDescription("Credential Store Token entity type");
if (!sharingRegistryServerHandler.isEntityTypeExists(entityType.getDomainId(), entityType.getEntityTypeId()))
sharingRegistryServerHandler.createEntityType(entityType);
//Creating Permission Types for each domain
PermissionType permissionType = new PermissionType();
permissionType.setPermissionTypeId(domain.getDomainId()+":READ");
permissionType.setDomainId(domain.getDomainId());
permissionType.setName("READ");
permissionType.setDescription("Read permission type");
if (!sharingRegistryServerHandler.isPermissionExists(permissionType.getDomainId(), permissionType.getPermissionTypeId()))
sharingRegistryServerHandler.createPermissionType(permissionType);
permissionType = new PermissionType();
permissionType.setPermissionTypeId(domain.getDomainId()+":WRITE");
permissionType.setDomainId(domain.getDomainId());
permissionType.setName("WRITE");
permissionType.setDescription("Write permission type");
if (!sharingRegistryServerHandler.isPermissionExists(permissionType.getDomainId(), permissionType.getPermissionTypeId()))
sharingRegistryServerHandler.createPermissionType(permissionType);
permissionType = new PermissionType();
permissionType.setPermissionTypeId(domain.getDomainId()+":MANAGE_SHARING");
permissionType.setDomainId(domain.getDomainId());
permissionType.setName("MANAGE_SHARING");
permissionType.setDescription("Sharing permission type");
if (!sharingRegistryServerHandler.isPermissionExists(permissionType.getDomainId(), permissionType.getPermissionTypeId()))
sharingRegistryServerHandler.createPermissionType(permissionType);
}catch (Exception ex){
ex.printStackTrace();
}
}
//Creating user entries
query = "SELECT * FROM USERS" + gatewayWhereClause;
statement = expCatConnection.createStatement();
rs = statement.executeQuery(query);
while(rs.next()){
try{
User user = new User();
user.setUserId(rs.getString("AIRAVATA_INTERNAL_USER_ID"));
user.setDomainId(rs.getString("GATEWAY_ID"));
user.setUserName(rs.getString("USER_NAME"));
if (!sharingRegistryServerHandler.isUserExists(user.getDomainId(), user.getUserId()))
sharingRegistryServerHandler.createUser(user);
}catch (Exception ex){
ex.printStackTrace();
}
}
//Map to reuse the domain ID and owner for creating application-deployment entities
Map<String, String> domainOwnerMap = new HashMap<>();
Map<String, GatewayGroups> gatewayGroupsMap = new HashMap<>();
//Creating the gateway groups
List<Domain> domainList = sharingRegistryServerHandler.getDomains(0, -1);
final RegistryService.Client registryServiceClient = getRegistryServiceClient();
for (Domain domain : domainList) {
// If we're only running migration for gatewayId, then skip other gateways
if (gatewayId != null && !gatewayId.equals(domain.getDomainId())) {
continue;
}
String ownerId = getAdminOwnerUser(domain, sharingRegistryServerHandler, credentialStoreServiceClient, registryServiceClient);
if (ownerId != null) {
domainOwnerMap.put(domain.getDomainId(), ownerId);
} else {
continue;
}
if (registryServiceClient.isGatewayGroupsExists(domain.getDomainId())) {
GatewayGroups gatewayGroups = registryServiceClient.getGatewayGroups(domain.getDomainId());
gatewayGroupsMap.put(domain.getDomainId(), gatewayGroups);
} else {
GatewayGroups gatewayGroups = migrateRolesToGatewayGroups(domain, ownerId, sharingRegistryServerHandler, registryServiceClient);
gatewayGroupsMap.put(domain.getDomainId(), gatewayGroups);
}
//find all the active users in keycloak that do not exist in sharing registry service and migrate them to the database
AuthzToken authzToken_of_management_user = getManagementUsersAccessToken(domain.getDomainId());
List<UserProfile> missingUsers = getUsersToMigrate(sharingRegistryServerHandler, iamAdminServiceClient, authzToken_of_management_user, null, domain.getDomainId());
migrateKeycloakUsersToGateway(iamAdminServiceClient, authzToken_of_management_user, missingUsers);
addUsersToGroups(sharingRegistryServerHandler, missingUsers, gatewayGroupsMap.get(domain.getDomainId()), domain.getDomainId());
}
//Creating project entries
query = "SELECT * FROM PROJECT" + gatewayWhereClause;
statement = expCatConnection.createStatement();
rs = statement.executeQuery(query);
List<Entity> projectEntities = new ArrayList<>();
while(rs.next()){
try{
Entity entity = new Entity();
entity.setEntityId(rs.getString("PROJECT_ID"));
entity.setDomainId(rs.getString("GATEWAY_ID"));
entity.setEntityTypeId(rs.getString("GATEWAY_ID") + ":PROJECT");
entity.setOwnerId(rs.getString("USER_NAME") + "@" + rs.getString("GATEWAY_ID"));
entity.setName(rs.getString("PROJECT_NAME"));
entity.setDescription(rs.getString("DESCRIPTION"));
if(entity.getDescription() == null)
entity.setFullText(entity.getName());
else
entity.setFullText(entity.getName() + " " + entity.getDescription());
// Map<String, String> metadata = new HashMap<>();
// metadata.put("CREATION_TIME", rs.getDate("CREATION_TIME").toString());
projectEntities.add(entity);
}catch (Exception ex) {
ex.printStackTrace();
}
}
//Creating experiment entries
query = "SELECT * FROM EXPERIMENT" + gatewayWhereClause;
statement = expCatConnection.createStatement();
rs = statement.executeQuery(query);
List<Entity> experimentEntities = new ArrayList<>();
while(rs.next()){
try {
Entity entity = new Entity();
entity.setEntityId(rs.getString("EXPERIMENT_ID"));
entity.setDomainId(rs.getString("GATEWAY_ID"));
entity.setEntityTypeId(rs.getString("GATEWAY_ID") + ":EXPERIMENT");
entity.setOwnerId(rs.getString("USER_NAME") + "@" + rs.getString("GATEWAY_ID"));
entity.setParentEntityId(rs.getString("PROJECT_ID"));
entity.setName(rs.getString("EXPERIMENT_NAME"));
entity.setDescription(rs.getString("DESCRIPTION"));
if(entity.getDescription() == null)
entity.setFullText(entity.getName());
else
entity.setFullText(entity.getName() + " " + entity.getDescription());
// Map<String, String> metadata = new HashMap<>();
// metadata.put("CREATION_TIME", rs.getDate("CREATION_TIME").toString());
// metadata.put("EXPERIMENT_TYPE", rs.getString("EXPERIMENT_TYPE"));
// metadata.put("EXECUTION_ID", rs.getString("EXECUTION_ID"));
// metadata.put("GATEWAY_EXECUTION_ID", rs.getString("GATEWAY_EXECUTION_ID"));
// metadata.put("ENABLE_EMAIL_NOTIFICATION", rs.getString("ENABLE_EMAIL_NOTIFICATION"));
// metadata.put("EMAIL_ADDRESSES", rs.getString("EMAIL_ADDRESSES"));
// metadata.put("GATEWAY_INSTANCE_ID", rs.getString("GATEWAY_INSTANCE_ID"));
// metadata.put("ARCHIVE", rs.getString("ARCHIVE"));
experimentEntities.add(entity);
}catch (Exception ex){
ex.printStackTrace();
}
}
for (Entity entity : projectEntities) {
if (!sharingRegistryServerHandler.isEntityExists(entity.getDomainId(), entity.getEntityId())) {
sharingRegistryServerHandler.createEntity(entity);
}
}
for (Entity entity : experimentEntities) {
if (!sharingRegistryServerHandler.isEntityExists(entity.getDomainId(), entity.getEntityId())) {
if (!sharingRegistryServerHandler.isEntityExists(entity.getDomainId(), entity.getParentEntityId())) {
System.out.println("Warning: project entity does exist for experiment entity " + entity.getEntityId() + " in gateway " + entity.getDomainId());
continue;
} else {
sharingRegistryServerHandler.createEntity(entity);
}
}
if (gatewayGroupsMap.containsKey(entity.getDomainId())) {
shareEntityWithAdminGatewayGroups(sharingRegistryServerHandler, entity, gatewayGroupsMap.get(entity.getDomainId()), false);
} else {
System.out.println("Warning: no Admin gateway groups to share experiment entity " + entity.getEntityId() + " in gateway " + entity.getDomainId());
}
}
//Creating application deployment entries
for (String domainID : domainOwnerMap.keySet()) {
GatewayGroups gatewayGroups = gatewayGroupsMap.get(domainID);
List<ApplicationDeploymentDescription> applicationDeploymentDescriptionList = registryServiceClient.getAllApplicationDeployments(domainID);
for (ApplicationDeploymentDescription description : applicationDeploymentDescriptionList) {
Entity entity = new Entity();
entity.setEntityId(description.getAppDeploymentId());
entity.setDomainId(domainID);
entity.setEntityTypeId(entity.getDomainId() + ":" + ResourceType.APPLICATION_DEPLOYMENT.name());
entity.setOwnerId(domainOwnerMap.get(domainID));
entity.setName(description.getAppDeploymentId());
entity.setDescription(description.getAppDeploymentDescription());
if (entity.getDescription() == null)
entity.setDescription(entity.getName());
else
entity.setFullText(entity.getName() + " " + entity.getDescription());
if (!sharingRegistryServerHandler.isEntityExists(entity.getDomainId(), entity.getEntityId()))
sharingRegistryServerHandler.createEntity(entity);
shareEntityWithGatewayGroups(sharingRegistryServerHandler, entity, gatewayGroups, false);
}
}
// Migrating from GatewayResourceProfile to GroupResourceProfile
for (String domainID : domainOwnerMap.keySet()) {
GatewayGroups gatewayGroups = gatewayGroupsMap.get(domainID);
if (needsGroupResourceProfileMigration(domainID, domainOwnerMap.get(domainID), registryServiceClient, sharingRegistryServerHandler)) {
GroupResourceProfile groupResourceProfile = migrateGatewayResourceProfileToGroupResourceProfile(domainID, registryServiceClient);
// create GroupResourceProfile entity in sharing registry
Entity entity = new Entity();
entity.setEntityId(groupResourceProfile.getGroupResourceProfileId());
entity.setDomainId(domainID);
entity.setEntityTypeId(entity.getDomainId() + ":" + ResourceType.GROUP_RESOURCE_PROFILE.name());
entity.setOwnerId(domainOwnerMap.get(domainID));
entity.setName(groupResourceProfile.getGroupResourceProfileName());
entity.setDescription(groupResourceProfile.getGroupResourceProfileName() + " Group Resource Profile");
if (!sharingRegistryServerHandler.isEntityExists(entity.getDomainId(), entity.getEntityId()))
sharingRegistryServerHandler.createEntity(entity);
shareEntityWithGatewayGroups(sharingRegistryServerHandler, entity, gatewayGroups, false);
}
}
// Creating credential store token entries (GATEWAY SSH tokens)
for (String domainID : domainOwnerMap.keySet()) {
List<CredentialSummary> gatewayCredentialSummaries = credentialStoreServiceClient.getAllCredentialSummaryForGateway(SummaryType.SSH, domainID);
for (CredentialSummary credentialSummary : gatewayCredentialSummaries) {
Entity entity = new Entity();
entity.setEntityId(credentialSummary.getToken());
entity.setDomainId(domainID);
entity.setEntityTypeId(entity.getDomainId() + ":" + ResourceType.CREDENTIAL_TOKEN.name());
entity.setOwnerId(domainOwnerMap.get(domainID));
entity.setName(credentialSummary.getToken());
entity.setDescription(maxLengthString(credentialSummary.getDescription(), 255));
if (!sharingRegistryServerHandler.isEntityExists(entity.getDomainId(), entity.getEntityId()))
sharingRegistryServerHandler.createEntity(entity);
if (gatewayGroupsMap.containsKey(entity.getDomainId())) {
shareEntityWithAdminGatewayGroups(sharingRegistryServerHandler, entity, gatewayGroupsMap.get(entity.getDomainId()), false);
}
}
}
// Creating credential store token entries (USER SSH tokens)
for (String domainID : domainOwnerMap.keySet()) {
List<User> sharingUsers = sharingRegistryServerHandler.getUsers(domainID, 0, Integer.MAX_VALUE);
for (User sharingUser : sharingUsers) {
String userId = sharingUser.getUserId();
if (!userId.endsWith("@" + domainID)) {
System.out.println("Skipping credentials for user " + userId + " since sharing user id is improperly formed");
continue;
}
String username = userId.substring(0, userId.lastIndexOf("@"));
List<CredentialSummary> gatewayCredentialSummaries = credentialStoreServiceClient.getAllCredentialSummaryForUserInGateway(SummaryType.SSH, domainID, username);
for (CredentialSummary credentialSummary : gatewayCredentialSummaries) {
Entity entity = new Entity();
entity.setEntityId(credentialSummary.getToken());
entity.setDomainId(domainID);
entity.setEntityTypeId(entity.getDomainId() + ":" + ResourceType.CREDENTIAL_TOKEN.name());
entity.setOwnerId(userId);
entity.setName(credentialSummary.getToken());
// Cap description length at max 255 characters
entity.setDescription(maxLengthString(credentialSummary.getDescription(), 255));
if (!sharingRegistryServerHandler.isEntityExists(entity.getDomainId(), entity.getEntityId()))
sharingRegistryServerHandler.createEntity(entity);
// Don't need to share USER SSH tokens with any group
}
}
}
// Creating credential store token entries (GATEWAY PWD tokens)
for (String domainID : domainOwnerMap.keySet()) {
Map<String, String> gatewayPasswords = credentialStoreServiceClient.getAllPWDCredentialsForGateway(domainID);
for (Map.Entry<String, String> gatewayPasswordEntry : gatewayPasswords.entrySet()) {
Entity entity = new Entity();
entity.setEntityId(gatewayPasswordEntry.getKey());
entity.setDomainId(domainID);
entity.setEntityTypeId(entity.getDomainId() + ":" + ResourceType.CREDENTIAL_TOKEN.name());
entity.setOwnerId(domainOwnerMap.get(domainID));
entity.setName(gatewayPasswordEntry.getKey());
entity.setDescription(maxLengthString(gatewayPasswordEntry.getValue(), 255));
if (!sharingRegistryServerHandler.isEntityExists(entity.getDomainId(), entity.getEntityId()))
sharingRegistryServerHandler.createEntity(entity);
if (gatewayGroupsMap.containsKey(entity.getDomainId())) {
shareEntityWithAdminGatewayGroups(sharingRegistryServerHandler, entity, gatewayGroupsMap.get(entity.getDomainId()), false);
}
}
}
expCatConnection.close();
System.out.println("Completed!");
System.exit(0);
}