import copy import datetime import json import logging from pathlib import Path from urllib.parse import quote from airavata.model.appcatalog.appdeployment.ttypes import ( ApplicationDeploymentDescription, ApplicationModule, CommandObject, SetEnvPaths ) from airavata.model.appcatalog.appinterface.ttypes import ( ApplicationInterfaceDescription ) from airavata.model.appcatalog.computeresource.ttypes import ( BatchQueue, ComputeResourceDescription ) from airavata.model.appcatalog.gatewayprofile.ttypes import ( GatewayResourceProfile, StoragePreference ) from airavata.model.appcatalog.groupresourceprofile.ttypes import ( ComputeResourceReservation, GroupComputeResourcePreference, GroupResourceProfile ) from airavata.model.appcatalog.parser.ttypes import Parser from airavata.model.appcatalog.storageresource.ttypes import ( StorageResourceDescription ) from airavata.model.application.io.ttypes import ( InputDataObjectType, OutputDataObjectType ) from airavata.model.credential.store.ttypes import ( CredentialSummary, SummaryType ) from airavata.model.data.replica.ttypes import ( DataProductModel, DataReplicaLocationModel ) from airavata.model.experiment.ttypes import ( ExperimentModel, ExperimentStatistics, ExperimentSummaryModel ) from airavata.model.group.ttypes import GroupModel, ResourcePermissionType from airavata.model.job.ttypes import JobModel from airavata.model.status.ttypes import ( ExperimentState, ExperimentStatus, ProcessStatus ) from airavata.model.user.ttypes import UserProfile from airavata.model.workspace.ttypes import ( Notification, NotificationPriority, Project ) from airavata_django_portal_sdk import ( experiment_util, queue_settings_calculators, user_storage ) from django.conf import settings from django.contrib.auth import get_user_model from django.urls import reverse from rest_framework import serializers from . import models, thrift_utils, view_utils log = logging.getLogger(__name__) class FullyEncodedHyperlinkedIdentityField( serializers.HyperlinkedIdentityField): def get_url(self, obj, view_name, request, format): if hasattr(obj, self.lookup_field): lookup_value = getattr(obj, self.lookup_field) else: lookup_value = obj.get(self.lookup_field) try: encoded_lookup_value = quote(lookup_value, safe="") except Exception: log.warning( "Failed to encode lookup_value [{}] for lookup_field " "[{}] of object [{}]".format( lookup_value, self.lookup_field, obj)) raise # Bit of a hack. Django's URL reversing does URL encoding but it # doesn't encode all characters including some like '/' that are used # in URL mappings. kwargs = {self.lookup_url_kwarg: "__PLACEHOLDER__"} url = self.reverse(view_name, kwargs=kwargs, request=request, format=format) return url.replace("__PLACEHOLDER__", encoded_lookup_value) class UTCPosixTimestampDateTimeField(serializers.DateTimeField): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.default = self.current_time_ms self.initial = self.initial_value self.required = False def to_representation(self, obj): # Create datetime instance from milliseconds that is aware of timezon dt = datetime.datetime.fromtimestamp(obj / 1000, datetime.timezone.utc) return super().to_representation(dt) def to_internal_value(self, data): dt = super().to_internal_value(data) return int(dt.timestamp() * 1000) def initial_value(self): return self.to_representation(self.current_time_ms()) def current_time_ms(self): return int(datetime.datetime.utcnow().timestamp() * 1000) class StoredJSONField(serializers.JSONField): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def to_representation(self, value): try: if value: return json.loads(value) else: return value except Exception: return value def to_internal_value(self, data): try: return json.dumps(data) except (TypeError, ValueError): self.fail('invalid') class OrderedListField(serializers.ListField): def __init__(self, *args, **kwargs): self.order_by = kwargs.pop('order_by', None) super().__init__(*args, **kwargs) def to_representation(self, instance): rep = super().to_representation(instance) if rep is not None: rep.sort(key=lambda item: item[self.order_by]) return rep def to_internal_value(self, data): validated_data = super().to_internal_value(data) # Update order field based on order in array items = validated_data if validated_data else [] for i in range(len(items)): items[i][self.order_by] = i return validated_data class GroupSerializer(thrift_utils.create_serializer_class(GroupModel)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:group-detail', lookup_field='id', lookup_url_kwarg='group_id') isAdmin = serializers.SerializerMethodField() isOwner = serializers.SerializerMethodField() isMember = serializers.SerializerMethodField() isGatewayAdminsGroup = serializers.SerializerMethodField() isReadOnlyGatewayAdminsGroup = serializers.SerializerMethodField() isDefaultGatewayUsersGroup = serializers.SerializerMethodField() class Meta: required = ('name',) read_only = ('ownerId',) def create(self, validated_data): group = super().create(validated_data) group.ownerId = self.context['request'].user.username + \ "@" + settings.GATEWAY_ID return group def update(self, instance, validated_data): instance.name = validated_data.get('name', instance.name) instance.description = validated_data.get( 'description', instance.description) # Calculate added and removed members old_members = set(instance.members) new_members = set(validated_data.get('members', instance.members)) removed_members = old_members - new_members added_members = new_members - old_members instance._removed_members = list(removed_members) instance._added_members = list(added_members) instance.members = validated_data.get('members', instance.members) # Calculate added and removed admins old_admins = set(instance.admins) new_admins = set(validated_data.get('admins', instance.admins)) removed_admins = old_admins - new_admins added_admins = new_admins - old_admins instance._removed_admins = list(removed_admins) instance._added_admins = list(added_admins) instance.admins = validated_data.get('admins', instance.admins) # Add new admins that aren't members to the added_members list instance._added_members.extend(list(added_admins - new_members)) instance.members.extend(list(added_admins - new_members)) return instance def get_isAdmin(self, group): request = self.context['request'] return request.profile_service['group_manager'].hasAdminAccess( request.authz_token, group.id, request.user.username + "@" + settings.GATEWAY_ID) def get_isOwner(self, group): request = self.context['request'] return group.ownerId == (request.user.username + "@" + settings.GATEWAY_ID) def get_isMember(self, group): request = self.context['request'] username = request.user.username + "@" + settings.GATEWAY_ID return group.members and username in group.members def get_isGatewayAdminsGroup(self, group): return group.id == self._gateway_groups()['adminsGroupId'] def get_isReadOnlyGatewayAdminsGroup(self, group): return group.id == self._gateway_groups()['readOnlyAdminsGroupId'] def get_isDefaultGatewayUsersGroup(self, group): return group.id == self._gateway_groups()['defaultGatewayUsersGroupId'] def _gateway_groups(self): request = self.context['request'] # gateway_groups_middleware sets this session variable if 'GATEWAY_GROUPS' in request.session: return request.session['GATEWAY_GROUPS'] else: gateway_groups = request.airavata_client.getGatewayGroups( request.authz_token) return copy.deepcopy(gateway_groups.__dict__) class ProjectSerializer( thrift_utils.create_serializer_class(Project)): class Meta: required = ('name',) read_only = ('owner', 'gatewayId') url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:project-detail', lookup_field='projectID', lookup_url_kwarg='project_id') experiments = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:project-experiments', lookup_field='projectID', lookup_url_kwarg='project_id') creationTime = UTCPosixTimestampDateTimeField(allow_null=True) userHasWriteAccess = serializers.SerializerMethodField() isOwner = serializers.SerializerMethodField() def create(self, validated_data): return Project(**validated_data) def update(self, instance, validated_data): instance.name = validated_data.get('name', instance.name) instance.description = validated_data.get( 'description', instance.description) return instance def get_userHasWriteAccess(self, project): request = self.context['request'] return request.airavata_client.userHasAccess( request.authz_token, project.projectID, ResourcePermissionType.WRITE) def get_isOwner(self, project): request = self.context['request'] return project.owner == request.user.username class ApplicationModuleSerializer( thrift_utils.create_serializer_class(ApplicationModule)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:application-detail', lookup_field='appModuleId', lookup_url_kwarg='app_module_id') applicationInterface = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:application-application-interface', lookup_field='appModuleId', lookup_url_kwarg='app_module_id') applicationDeployments = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:application-application-deployments', lookup_field='appModuleId', lookup_url_kwarg='app_module_id') userHasWriteAccess = serializers.SerializerMethodField() class Meta: required = ('appModuleName',) def get_userHasWriteAccess(self, appDeployment): request = self.context['request'] return request.is_gateway_admin class InputDataObjectTypeSerializer( thrift_utils.create_serializer_class(InputDataObjectType)): metaData = StoredJSONField(required=False, allow_null=True) class Meta: required = ('name',) class OutputDataObjectTypeSerializer( thrift_utils.create_serializer_class(OutputDataObjectType)): metaData = StoredJSONField(required=False, allow_null=True) class Meta: required = ('name',) class ApplicationInterfaceDescriptionSerializer( thrift_utils.create_serializer_class(ApplicationInterfaceDescription)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:application-interface-detail', lookup_field='applicationInterfaceId', lookup_url_kwarg='app_interface_id') applicationInputs = OrderedListField( order_by='inputOrder', child=InputDataObjectTypeSerializer(), allow_null=True) applicationOutputs = OutputDataObjectTypeSerializer(many=True) userHasWriteAccess = serializers.SerializerMethodField() showQueueSettings = serializers.BooleanField(required=False) queueSettingsCalculatorId = serializers.CharField(allow_null=True, required=False) def to_representation(self, instance): representation = super().to_representation(instance) application_module_id = instance.applicationModules[0] application_settings, created = models.ApplicationSettings.objects.get_or_create( application_module_id=application_module_id) representation["showQueueSettings"] = application_settings.show_queue_settings # check that queue_settings_calculator_id exists if queue_settings_calculators.exists(application_settings.queue_settings_calculator_id): representation["queueSettingsCalculatorId"] = application_settings.queue_settings_calculator_id return representation def create(self, validated_data): showQueueSettings = validated_data.pop("showQueueSettings", True) queueSettingsCalculatorId = validated_data.pop("queueSettingsCalculatorId", None) application_interface = super().create(validated_data) application_module_id = application_interface.applicationModules[0] models.ApplicationSettings.objects.update_or_create( application_module_id=application_module_id, defaults={"show_queue_settings": showQueueSettings, "queue_settings_calculator_id": queueSettingsCalculatorId} ) return application_interface def update(self, instance, validated_data): defaults = {} if "showQueueSettings" in validated_data: defaults["show_queue_settings"] = validated_data.pop("showQueueSettings") if "queueSettingsCalculatorId" in validated_data: defaults["queue_settings_calculator_id"] = validated_data.pop("queueSettingsCalculatorId") application_interface = super().update(instance, validated_data) application_module_id = application_interface.applicationModules[0] models.ApplicationSettings.objects.update_or_create( application_module_id=application_module_id, defaults=defaults ) return application_interface def get_userHasWriteAccess(self, appDeployment): request = self.context['request'] return request.is_gateway_admin class CommandObjectSerializer( thrift_utils.create_serializer_class(CommandObject)): pass class SetEnvPathsSerializer( thrift_utils.create_serializer_class(SetEnvPaths)): pass class ApplicationDeploymentDescriptionSerializer( thrift_utils.create_serializer_class( ApplicationDeploymentDescription)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:application-deployment-detail', lookup_field='appDeploymentId', lookup_url_kwarg='app_deployment_id') # Default values returned in these results have been overridden with app # deployment defaults for any that exist queues = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:application-deployment-queues', lookup_field='appDeploymentId', lookup_url_kwarg='app_deployment_id') userHasWriteAccess = serializers.SerializerMethodField() moduleLoadCmds = OrderedListField( order_by='commandOrder', child=CommandObjectSerializer(), allow_null=True) preJobCommands = OrderedListField( order_by='commandOrder', child=CommandObjectSerializer(), allow_null=True) postJobCommands = OrderedListField( order_by='commandOrder', child=CommandObjectSerializer(), allow_null=True) libPrependPaths = OrderedListField( order_by='envPathOrder', child=SetEnvPathsSerializer(), allow_null=True) libAppendPaths = OrderedListField( order_by='envPathOrder', child=SetEnvPathsSerializer(), allow_null=True) setEnvironment = OrderedListField( order_by='envPathOrder', child=SetEnvPathsSerializer(), allow_null=True) def get_userHasWriteAccess(self, appDeployment): request = self.context['request'] return request.airavata_client.userHasAccess( request.authz_token, appDeployment.appDeploymentId, ResourcePermissionType.WRITE) class ComputeResourceDescriptionSerializer( thrift_utils.create_serializer_class(ComputeResourceDescription)): pass class BatchQueueSerializer(thrift_utils.create_serializer_class(BatchQueue)): pass class ExperimentStatusSerializer( thrift_utils.create_serializer_class(ExperimentStatus)): timeOfStateChange = UTCPosixTimestampDateTimeField() class ProcessStatusSerializer( thrift_utils.create_serializer_class(ProcessStatus)): timeOfStateChange = UTCPosixTimestampDateTimeField() class ExperimentSerializer( thrift_utils.create_serializer_class(ExperimentModel)): class Meta: required = ('projectId', 'experimentType', 'experimentName') read_only = ('userName', 'gatewayId') url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:experiment-detail', lookup_field='experimentId', lookup_url_kwarg='experiment_id') full_experiment = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:full-experiment-detail', lookup_field='experimentId', lookup_url_kwarg='experiment_id') project = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:project-detail', lookup_field='projectId', lookup_url_kwarg='project_id') jobs = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:experiment-jobs', lookup_field='experimentId', lookup_url_kwarg='experiment_id') shared_entity = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:shared-entity-detail', lookup_field='experimentId', lookup_url_kwarg='entity_id') experimentInputs = OrderedListField( order_by='inputOrder', child=InputDataObjectTypeSerializer(), allow_null=True) experimentOutputs = serializers.ListField( child=OutputDataObjectTypeSerializer(), allow_null=True) creationTime = UTCPosixTimestampDateTimeField(allow_null=True) experimentStatus = ExperimentStatusSerializer(many=True, allow_null=True) userHasWriteAccess = serializers.SerializerMethodField() def get_userHasWriteAccess(self, experiment): request = self.context['request'] return request.airavata_client.userHasAccess( request.authz_token, experiment.experimentId, ResourcePermissionType.WRITE) def to_representation(self, experiment): result = super().to_representation(experiment) self._add_intermediate_output_information(experiment, result) return result def _add_intermediate_output_information(self, experiment, representation): request = self.context['request'] # If experiment is EXECUTING, add intermediateOutput information to # experiment outputs if (experiment.experimentStatus and experiment.experimentStatus[-1].state == ExperimentState.EXECUTING): for output in representation["experimentOutputs"]: output["intermediateOutput"] = {"processStatus": None} try: can_fetch = experiment_util.intermediate_output.can_fetch_intermediate_output(request, experiment, output["name"]) output["intermediateOutput"]["canFetch"] = can_fetch process_status = experiment_util.intermediate_output.get_intermediate_output_process_status( request, experiment, output["name"]) if process_status: serializer = ProcessStatusSerializer( process_status, context={'request': request}) output["intermediateOutput"]["processStatus"] = serializer.data data_products = experiment_util.intermediate_output.get_intermediate_output_data_products( request, experiment=experiment, output_name=output["name"]) data_product_serializer = DataProductSerializer( data_products, context={'request': request}, many=True) output["intermediateOutput"]["dataProducts"] = data_product_serializer.data except Exception: log.debug("Failed to get intermediate output status", exc_info=True) class DataReplicaLocationSerializer( thrift_utils.create_serializer_class(DataReplicaLocationModel)): creationTime = UTCPosixTimestampDateTimeField() lastModifiedTime = UTCPosixTimestampDateTimeField() class DataProductSerializer( thrift_utils.create_serializer_class(DataProductModel)): creationTime = UTCPosixTimestampDateTimeField() modifiedTime = UTCPosixTimestampDateTimeField() lastModifiedTime = UTCPosixTimestampDateTimeField() replicaLocations = DataReplicaLocationSerializer(many=True) downloadURL = serializers.SerializerMethodField() isInputFileUpload = serializers.SerializerMethodField() filesize = serializers.SerializerMethodField() userHasWriteAccess = serializers.SerializerMethodField() def get_downloadURL(self, data_product): """Getter for downloadURL field. Returns None if file is not available.""" request = self.context['request'] if user_storage.exists(request, data_product): return user_storage.get_lazy_download_url(request, data_product) else: return None def get_isInputFileUpload(self, data_product): """Return True if this is an uploaded input file.""" request = self.context['request'] return user_storage.is_input_file(request, data_product) def get_filesize(self, data_product): request = self.context['request'] # For backwards compatibility with older user_storage, can be eventually removed if hasattr(user_storage, 'get_data_product_metadata') and user_storage.exists(request, data_product): metadata = user_storage.get_data_product_metadata(request, data_product) return metadata['size'] else: return 0 def get_userHasWriteAccess(self, data_product: DataProductModel): request = self.context['request'] if user_storage.exists(request, data_product): file_metadata = user_storage.get_data_product_metadata(request, data_product=data_product) # In remote API mode, "userHasWriteAccess" is returned so we just pass it through here if "userHasWriteAccess" in file_metadata: return file_metadata["userHasWriteAccess"] else: path = file_metadata["path"] shared_path = view_utils.is_shared_path(path) if shared_path: # Only admins can edit files/directories in a shared directory return request.is_gateway_admin return True else: return False # TODO move this into airavata_sdk? class FullExperiment: """Experiment with referenced data models.""" def __init__(self, experimentModel, project=None, outputDataProducts=None, inputDataProducts=None, applicationModule=None, computeResource=None, jobDetails=None, outputViews=None): self.experiment = experimentModel self.experimentId = experimentModel.experimentId self.project = project self.outputDataProducts = outputDataProducts self.inputDataProducts = inputDataProducts self.applicationModule = applicationModule self.computeResource = computeResource self.jobDetails = jobDetails self.outputViews = outputViews class JobSerializer(thrift_utils.create_serializer_class(JobModel)): creationTime = UTCPosixTimestampDateTimeField() class FullExperimentSerializer(serializers.Serializer): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:full-experiment-detail', lookup_field='experimentId', lookup_url_kwarg='experiment_id') experiment = ExperimentSerializer() experimentId = serializers.CharField(read_only=True) outputDataProducts = DataProductSerializer(many=True, read_only=True) inputDataProducts = DataProductSerializer(many=True, read_only=True) applicationModule = ApplicationModuleSerializer(read_only=True) computeResource = ComputeResourceDescriptionSerializer(read_only=True) project = ProjectSerializer(read_only=True) jobDetails = JobSerializer(many=True, read_only=True) outputViews = serializers.DictField(read_only=True) def create(self, validated_data): raise Exception("Not implemented") def update(self, instance, validated_data): raise Exception("Not implemented") class BaseExperimentSummarySerializer( thrift_utils.create_serializer_class(ExperimentSummaryModel)): creationTime = UTCPosixTimestampDateTimeField() statusUpdateTime = UTCPosixTimestampDateTimeField() url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:experiment-detail', lookup_field='experimentId', lookup_url_kwarg='experiment_id') project = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:project-detail', lookup_field='projectId', lookup_url_kwarg='project_id') class ExperimentSummarySerializer(BaseExperimentSummarySerializer): userHasWriteAccess = serializers.SerializerMethodField() def get_userHasWriteAccess(self, experiment): request = self.context['request'] return request.airavata_client.userHasAccess( request.authz_token, experiment.experimentId, ResourcePermissionType.WRITE) class UserProfileSerializer( thrift_utils.create_serializer_class(UserProfile)): creationTime = UTCPosixTimestampDateTimeField() lastAccessTime = UTCPosixTimestampDateTimeField() class ComputeResourceReservationSerializer( thrift_utils.create_serializer_class(ComputeResourceReservation)): startTime = UTCPosixTimestampDateTimeField(allow_null=True) endTime = UTCPosixTimestampDateTimeField(allow_null=True) class GroupComputeResourcePreferenceSerializer( thrift_utils.create_serializer_class(GroupComputeResourcePreference)): reservations = ComputeResourceReservationSerializer(many=True) class GroupResourceProfileSerializer( thrift_utils.create_serializer_class(GroupResourceProfile)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:group-resource-profile-detail', lookup_field='groupResourceProfileId', lookup_url_kwarg='group_resource_profile_id') creationTime = UTCPosixTimestampDateTimeField(allow_null=True) updatedTime = UTCPosixTimestampDateTimeField(allow_null=True) userHasWriteAccess = serializers.SerializerMethodField() computePreferences = GroupComputeResourcePreferenceSerializer(many=True) class Meta: required = ('groupResourceProfileName',) def update(self, instance, validated_data): result = super().update(instance, validated_data) result._removed_compute_resource_preferences = [] result._removed_compute_resource_policies = [] result._removed_batch_queue_resource_policies = [] # Find all compute resource preferences that were removed for compute_resource_preference in instance.computePreferences: existing_compute_resource_preference = next( (pref for pref in result.computePreferences if pref.computeResourceId == compute_resource_preference.computeResourceId), None) if not existing_compute_resource_preference: result._removed_compute_resource_preferences.append( compute_resource_preference) # Find all compute resource policies that were removed for compute_resource_policy in instance.computeResourcePolicies: existing_compute_resource_policy = next( (pol for pol in result.computeResourcePolicies if pol.resourcePolicyId == compute_resource_policy.resourcePolicyId), None) if not existing_compute_resource_policy: result._removed_compute_resource_policies.append( compute_resource_policy) # Find all batch queue resource policies that were removed for batch_queue_resource_policy in instance.batchQueueResourcePolicies: existing_batch_queue_resource_policy_for_update = next( (bq for bq in result.batchQueueResourcePolicies if bq.resourcePolicyId == batch_queue_resource_policy.resourcePolicyId), None) if not existing_batch_queue_resource_policy_for_update: result._removed_batch_queue_resource_policies.append( batch_queue_resource_policy) return result def get_userHasWriteAccess(self, groupResourceProfile): request = self.context['request'] write_access = request.airavata_client.userHasAccess( request.authz_token, groupResourceProfile.groupResourceProfileId, ResourcePermissionType.WRITE) if not write_access: return False # Check that user has READ access to all tokens in this # GroupResourceProfile tokens = set([groupResourceProfile.defaultCredentialStoreToken] + [cp.resourceSpecificCredentialStoreToken for cp in groupResourceProfile.computePreferences]) def check_token(token): return token is None or request.airavata_client.userHasAccess( request.authz_token, token, ResourcePermissionType.READ) return all(map(check_token, tokens)) class UserPermissionSerializer(serializers.Serializer): user = UserProfileSerializer() permissionType = serializers.IntegerField() class GroupPermissionSerializer(serializers.Serializer): group = GroupSerializer() permissionType = serializers.IntegerField() class SharedEntitySerializer(serializers.Serializer): entityId = serializers.CharField(read_only=True) userPermissions = UserPermissionSerializer(many=True) groupPermissions = GroupPermissionSerializer(many=True) owner = UserProfileSerializer(read_only=True) isOwner = serializers.SerializerMethodField() hasSharingPermission = serializers.SerializerMethodField() def create(self, validated_data): raise Exception("Not implemented") def update(self, instance, validated_data): # Compute lists of ids to grant/revoke READ/WRITE/MANAGE_SHARING # permission existing_user_permissions = { user['user'].airavataInternalUserId: user['permissionType'] for user in instance['userPermissions']} new_user_permissions = { user['user']['airavataInternalUserId']: user['permissionType'] for user in validated_data['userPermissions']} ( user_grant_read_permission, user_grant_write_permission, user_grant_manage_sharing_permission, user_revoke_read_permission, user_revoke_write_permission, user_revoke_manage_sharing_permission) = self._compute_all_revokes_and_grants( existing_user_permissions, new_user_permissions) existing_group_permissions = { group['group'].id: group['permissionType'] for group in instance['groupPermissions']} new_group_permissions = { group['group']['id']: group['permissionType'] for group in validated_data['groupPermissions']} ( group_grant_read_permission, group_grant_write_permission, group_grant_manage_sharing_permission, group_revoke_read_permission, group_revoke_write_permission, group_revoke_manage_sharing_permission) = self._compute_all_revokes_and_grants( existing_group_permissions, new_group_permissions) instance['_user_grant_read_permission'] = user_grant_read_permission instance['_user_grant_write_permission'] = user_grant_write_permission instance['_user_grant_manage_sharing_permission'] = user_grant_manage_sharing_permission instance['_user_revoke_read_permission'] = user_revoke_read_permission instance['_user_revoke_write_permission'] = user_revoke_write_permission instance['_user_revoke_manage_sharing_permission'] = user_revoke_manage_sharing_permission instance['_group_grant_read_permission'] = group_grant_read_permission instance['_group_grant_write_permission'] = group_grant_write_permission instance['_group_grant_manage_sharing_permission'] = group_grant_manage_sharing_permission instance['_group_revoke_read_permission'] = group_revoke_read_permission instance['_group_revoke_write_permission'] = group_revoke_write_permission instance['_group_revoke_manage_sharing_permission'] = group_revoke_manage_sharing_permission instance['userPermissions'] = [ {'user': UserProfile(**data['user']), 'permissionType': data['permissionType']} for data in validated_data.get( 'userPermissions', instance['userPermissions'])] instance['groupPermissions'] = [ {'group': GroupModel(**data['group']), 'permissionType': data['permissionType']} for data in validated_data.get('groupPermissions', instance['groupPermissions'])] return instance def _compute_all_revokes_and_grants(self, existing_permissions, new_permissions): grant_read_permission = [] grant_write_permission = [] grant_manage_sharing_permission = [] revoke_read_permission = [] revoke_write_permission = [] revoke_manage_sharing_permission = [] # Union the two sets of user/group ids all_ids = existing_permissions.keys() | new_permissions.keys() for id in all_ids: revokes, grants = self._compute_revokes_and_grants( existing_permissions.get(id), new_permissions.get(id) ) if ResourcePermissionType.READ in revokes: revoke_read_permission.append(id) if ResourcePermissionType.WRITE in revokes: revoke_write_permission.append(id) if ResourcePermissionType.MANAGE_SHARING in revokes: revoke_manage_sharing_permission.append(id) if ResourcePermissionType.READ in grants: grant_read_permission.append(id) if ResourcePermissionType.WRITE in grants: grant_write_permission.append(id) if ResourcePermissionType.MANAGE_SHARING in grants: grant_manage_sharing_permission.append(id) return ( grant_read_permission, grant_write_permission, grant_manage_sharing_permission, revoke_read_permission, revoke_write_permission, revoke_manage_sharing_permission) def _compute_revokes_and_grants(self, current_permission=None, new_permission=None): read_permissions = set((ResourcePermissionType.READ,)) write_permissions = set((ResourcePermissionType.READ, ResourcePermissionType.WRITE)) manage_share_permissions = set( (ResourcePermissionType.READ, ResourcePermissionType.WRITE, ResourcePermissionType.MANAGE_SHARING)) current_permissions_set = set() new_permissions_set = set() if current_permission == ResourcePermissionType.READ: current_permissions_set = read_permissions elif current_permission == ResourcePermissionType.WRITE: current_permissions_set = write_permissions elif current_permission == ResourcePermissionType.MANAGE_SHARING: current_permissions_set = manage_share_permissions if new_permission == ResourcePermissionType.READ: new_permissions_set = read_permissions elif new_permission == ResourcePermissionType.WRITE: new_permissions_set = write_permissions elif new_permission == ResourcePermissionType.MANAGE_SHARING: new_permissions_set = manage_share_permissions # return tuple: permissions to revoke and permissions to grant return (current_permissions_set - new_permissions_set, new_permissions_set - current_permissions_set) def get_isOwner(self, shared_entity): request = self.context['request'] return shared_entity['owner'].userId == request.user.username def get_hasSharingPermission(self, shared_entity): request = self.context['request'] return request.airavata_client.userHasAccess( request.authz_token, shared_entity['entityId'], ResourcePermissionType.MANAGE_SHARING) class CredentialSummarySerializer( thrift_utils.create_serializer_class(CredentialSummary)): type = thrift_utils.ThriftEnumField(SummaryType) persistedTime = UTCPosixTimestampDateTimeField() userHasWriteAccess = serializers.SerializerMethodField() def get_userHasWriteAccess(self, credential_summary): request = self.context['request'] return request.airavata_client.userHasAccess( request.authz_token, credential_summary.token, ResourcePermissionType.WRITE) class StoragePreferenceSerializer( thrift_utils.create_serializer_class(StoragePreference)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:storage-preference-detail', lookup_field='storageResourceId', lookup_url_kwarg='storage_resource_id') def to_representation(self, instance): ret = super().to_representation(instance) # Convert empty string to null if ret['resourceSpecificCredentialStoreToken'] == '': ret['resourceSpecificCredentialStoreToken'] = None return ret class GatewayResourceProfileSerializer( thrift_utils.create_serializer_class(GatewayResourceProfile)): storagePreferences = StoragePreferenceSerializer(many=True) userHasWriteAccess = serializers.SerializerMethodField() def get_userHasWriteAccess(self, gatewayResourceProfile): request = self.context['request'] return request.is_gateway_admin class StorageResourceSerializer( thrift_utils.create_serializer_class(StorageResourceDescription)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:storage-resource-detail', lookup_field='storageResourceId', lookup_url_kwarg='storage_resource_id') creationTime = UTCPosixTimestampDateTimeField() updateTime = UTCPosixTimestampDateTimeField() class ParserSerializer(thrift_utils.create_serializer_class(Parser)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:parser-detail', lookup_field='id', lookup_url_kwarg='parser_id') class UserHasWriteAccessToPathSerializer(serializers.Serializer): userHasWriteAccess = serializers.SerializerMethodField() def get_userHasWriteAccess(self, instance): request = self.context['request'] # Special handling when using remote API to access user data storage if hasattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API'): if "userHasWriteAccess" in instance: return instance["userHasWriteAccess"] elif instance.get("isDir", False): path = Path(instance.get("path", "")) if path != Path(""): # get parent directory listing and use that to figure out if # there is write access to this directory directories, _ = user_storage.listdir(request, path.parent) for d in directories: if Path(d["path"]) == path: return d.get("userHasWriteAccess", False) return False else: # User always has write access on home directory return True else: return False is_shared_path = view_utils.is_shared_path(instance["path"]) if is_shared_path: return request.is_gateway_admin else: return True class UserStorageFileSerializer(UserHasWriteAccessToPathSerializer): name = serializers.CharField() downloadURL = serializers.SerializerMethodField() dataProductURI = serializers.CharField(source='data-product-uri') createdTime = serializers.DateTimeField(source='created_time') modifiedTime = serializers.DateTimeField(source='modified_time') mimeType = serializers.CharField(source='mime_type') size = serializers.IntegerField() hidden = serializers.BooleanField() def get_downloadURL(self, file): """Getter for downloadURL field.""" request = self.context['request'] return user_storage.get_lazy_download_url(request, data_product_uri=file['data-product-uri']) class UserStorageDirectorySerializer(UserHasWriteAccessToPathSerializer): name = serializers.CharField() path = serializers.CharField() createdTime = serializers.DateTimeField(source='created_time') modifiedTime = serializers.DateTimeField(source='modified_time') size = serializers.IntegerField() hidden = serializers.BooleanField() url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:user-storage-items', lookup_field='path', lookup_url_kwarg='path') isSharedDir = serializers.SerializerMethodField() def get_isSharedDir(self, directory): if "isSharedDir" in directory: return directory["isSharedDir"] return view_utils.is_shared_dir(directory["path"]) class UserStoragePathSerializer(UserHasWriteAccessToPathSerializer): isDir = serializers.BooleanField() directories = UserStorageDirectorySerializer(many=True) files = UserStorageFileSerializer(many=True) parts = serializers.ListField(child=serializers.CharField()) path = serializers.CharField(required=False) # uploaded is populated after a file upload uploaded = DataProductSerializer(read_only=True) # Fields for ExperimentStorageFileSerializer are the same as UserStorageFileSerializer ExperimentStorageFileSerializer = UserStorageFileSerializer class ExperimentStorageDirectorySerializer(serializers.Serializer): name = serializers.CharField() path = serializers.CharField() createdTime = serializers.DateTimeField(source='created_time') modifiedTime = serializers.DateTimeField(source='modified_time') size = serializers.IntegerField() url = serializers.SerializerMethodField() def get_url(self, dir): request = self.context['request'] return request.build_absolute_uri( reverse("django_airavata_api:experiment-storage-items", kwargs={ "experiment_id": dir['experiment_id'], "path": dir['path'] })) class ExperimentStoragePathSerializer(serializers.Serializer): isDir = serializers.BooleanField() directories = ExperimentStorageDirectorySerializer(many=True) files = ExperimentStorageFileSerializer(many=True) parts = serializers.ListField(child=serializers.CharField()) # ModelSerializers class ApplicationPreferencesSerializer(serializers.ModelSerializer): class Meta: model = models.ApplicationPreferences exclude = ('id', 'username', 'workspace_preferences') class WorkspacePreferencesSerializer(serializers.ModelSerializer): application_preferences = ApplicationPreferencesSerializer( source="applicationpreferences_set", many=True) class Meta: model = models.WorkspacePreferences exclude = ('username',) class IAMUserProfile(serializers.Serializer): airavataInternalUserId = serializers.CharField() userId = serializers.CharField() gatewayId = serializers.CharField() email = serializers.CharField() firstName = serializers.CharField() lastName = serializers.CharField() enabled = serializers.BooleanField() emailVerified = serializers.BooleanField() airavataUserProfileExists = serializers.BooleanField() creationTime = UTCPosixTimestampDateTimeField() groups = GroupSerializer(many=True) url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:iam-user-profile-detail', lookup_field='userId', lookup_url_kwarg='user_id') userHasWriteAccess = serializers.SerializerMethodField() newUsername = serializers.CharField(write_only=True, required=False) externalIDPUserInfo = serializers.SerializerMethodField() userProfileInvalidFields = serializers.SerializerMethodField() def update(self, instance, validated_data): existing_group_ids = [group.id for group in instance['groups']] new_group_ids = [group['id'] for group in validated_data['groups']] instance['_added_group_ids'] = list( set(new_group_ids) - set(existing_group_ids)) instance['_removed_group_ids'] = list( set(existing_group_ids) - set(new_group_ids)) return instance def get_userHasWriteAccess(self, userProfile): request = self.context['request'] return request.is_gateway_admin def get_externalIDPUserInfo(self, userProfile): result = {} try: if get_user_model().objects.filter(username=userProfile['userId']).exists(): django_user = get_user_model().objects.get(username=userProfile['userId']) claims = django_user.user_profile.idp_userinfo.all() if claims.exists(): result['idp_alias'] = claims.first().idp_alias result['userinfo'] = {} for claim in claims: result['userinfo'][claim.claim] = claim.value except Exception as e: log.warning(f"Failed to load idp_userinfo for {userProfile['userId']}", exc_info=e) return result def get_userProfileInvalidFields(self, userProfile): try: User = get_user_model() if User.objects.filter(username=userProfile['userId']).exists(): django_user = User.objects.get(username=userProfile['userId']) if hasattr(django_user, 'user_profile'): return django_user.user_profile.invalid_fields else: # For backwards compatibility, return True if no user_profile return [] except Exception as e: log.warning(f"Failed to get user_profile.invalid_fields for {userProfile['userId']}", exc_info=e) return [] class AckNotificationSerializer(serializers.ModelSerializer): class Meta: model = models.User_Notifications class NotificationSerializer(thrift_utils.create_serializer_class(Notification)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:manage-notifications-detail', lookup_field='notificationId', lookup_url_kwarg='notification_id') priority = thrift_utils.ThriftEnumField(NotificationPriority) creationTime = UTCPosixTimestampDateTimeField(allow_null=True) publishedTime = UTCPosixTimestampDateTimeField() expirationTime = UTCPosixTimestampDateTimeField() userHasWriteAccess = serializers.SerializerMethodField() showInDashboard = serializers.BooleanField(default=False) def get_userHasWriteAccess(self, userProfile): request = self.context['request'] return request.is_gateway_admin def validate(self, attrs): del attrs["showInDashboard"] return attrs def to_representation(self, notification): notification_extension_list = models.NotificationExtension.objects.filter( notification_id=notification.notificationId) setattr(notification, "showInDashboard", False if len(notification_extension_list) == 0 else notification_extension_list[0].showInDashboard) return super().to_representation(notification) def update_notification_extension(self, request, notification): if "showInDashboard" in request.data: existing_entries = models.NotificationExtension.objects.filter(notification_id=notification.notificationId) if len(existing_entries) > 0: existing_entries.update( showInDashboard=request.data["showInDashboard"] ) else: models.NotificationExtension.objects.create( notification_id=notification.notificationId, showInDashboard=request.data["showInDashboard"] ) class ExperimentStatisticsSerializer( thrift_utils.create_serializer_class(ExperimentStatistics)): allExperiments = BaseExperimentSummarySerializer(many=True) completedExperiments = BaseExperimentSummarySerializer(many=True) failedExperiments = BaseExperimentSummarySerializer(many=True) cancelledExperiments = BaseExperimentSummarySerializer(many=True) createdExperiments = BaseExperimentSummarySerializer(many=True) runningExperiments = BaseExperimentSummarySerializer(many=True) class UnverifiedEmailUserProfile(serializers.Serializer): userId = serializers.CharField() gatewayId = serializers.CharField() email = serializers.CharField() firstName = serializers.CharField() lastName = serializers.CharField() enabled = serializers.BooleanField() emailVerified = serializers.BooleanField() creationTime = UTCPosixTimestampDateTimeField() url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:unverified-email-user-profile-detail', lookup_field='userId', lookup_url_kwarg='user_id') userHasWriteAccess = serializers.SerializerMethodField() def get_userHasWriteAccess(self, userProfile): request = self.context['request'] return request.is_gateway_admin class LogRecordSerializer(serializers.Serializer): level = serializers.CharField() message = serializers.CharField() details = StoredJSONField() stacktrace = serializers.ListField(child=serializers.CharField()) class SettingsSerializer(serializers.Serializer): fileUploadMaxFileSize = serializers.IntegerField() tusEndpoint = serializers.CharField() pgaUrl = serializers.CharField() class QueueSettingsCalculatorSerializer(serializers.Serializer): id = serializers.CharField() name = serializers.CharField()