django_airavata/apps/api/views.py

import base64
import json
import logging
import os
import warnings
from datetime import datetime, timedelta

from airavata.model.appcatalog.computeresource.ttypes import (
    CloudJobSubmission,
    GlobusJobSubmission,
    LOCALSubmission,
    SSHJobSubmission,
    UnicoreJobSubmission
)
from airavata.model.application.io.ttypes import DataType
from airavata.model.credential.store.ttypes import SummaryType
from airavata.model.data.movement.ttypes import (
    GridFTPDataMovement,
    LOCALDataMovement,
    SCPDataMovement,
    UnicoreDataMovement
)
from airavata.model.experiment.ttypes import (
    ExperimentModel,
    ExperimentSearchFields
)
from airavata.model.group.ttypes import ResourcePermissionType
from airavata.model.user.ttypes import Status
from airavata_django_portal_sdk import (
    experiment_util,
    queue_settings_calculators,
    user_storage
)
from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django.http import Http404, HttpResponse, JsonResponse
from django.shortcuts import redirect
from django.urls import reverse
from django.views.decorators.gzip import gzip_page
from rest_framework import mixins, pagination, status
from rest_framework.decorators import action, api_view, permission_classes
from rest_framework.exceptions import ParseError
from rest_framework.permissions import IsAuthenticated
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.views import APIView

from django_airavata.apps.admin.models import UserDataArchiveEntry
from django_airavata.apps.api.view_utils import (
    APIBackedViewSet,
    APIResultIterator,
    APIResultPagination,
    DataProductSharedDirPermission,
    GenericAPIBackedViewSet,
    IsInAdminsGroupPermission,
    UserStorageSharedDirPermission
)
from django_airavata.apps.auth import iam_admin_client
from django_airavata.apps.auth.models import EmailVerification

from . import (
    exceptions,
    helpers,
    models,
    output_views,
    serializers,
    signals,
    thrift_utils,
    tus,
    view_utils
)

READ_PERMISSION_TYPE = '{}:READ'

log = logging.getLogger(__name__)


class GroupViewSet(APIBackedViewSet):
    serializer_class = serializers.GroupSerializer
    lookup_field = 'group_id'
    pagination_class = APIResultPagination
    pagination_viewname = 'django_airavata_api:group-list'

    def get_list(self):
        view = self

        class GroupResultsIterator(APIResultIterator):
            def get_results(self, limit=-1, offset=0):
                group_manager = view.request.profile_service['group_manager']
                groups = group_manager.getGroups(view.authz_token)
                end = offset + limit if limit > 0 else len(groups)
                return groups[offset:end] if groups else []

        return GroupResultsIterator()

    def get_instance(self, lookup_value):
        return self.request.profile_service['group_manager'].getGroup(
            self.authz_token, lookup_value)

    def perform_create(self, serializer):
        group = serializer.save()
        group_id = self.request.profile_service['group_manager'].createGroup(
            self.authz_token, group)
        group.id = group_id
        users_added_to_group = set(group.members) - {group.ownerId}
        self._send_users_added_to_group(users_added_to_group, group)

    def perform_update(self, serializer):
        group = serializer.save()
        group_manager_client = self.request.profile_service['group_manager']
        if len(group._added_members) > 0:
            group_manager_client.addUsersToGroup(
                self.authz_token, group._added_members, group.id)
            self._send_users_added_to_group(group._added_members, group)
        if len(group._removed_members) > 0:
            group_manager_client.removeUsersFromGroup(
                self.authz_token, group._removed_members, group.id)
        if len(group._added_admins) > 0:
            group_manager_client.addGroupAdmins(
                self.authz_token, group.id, group._added_admins)
        if len(group._removed_admins) > 0:
            group_manager_client.removeGroupAdmins(
                self.authz_token, group.id, group._removed_admins)
        group_manager_client.updateGroup(self.authz_token, group)

    def perform_destroy(self, group):
        group_manager_client = self.request.profile_service['group_manager']
        group_manager_client.deleteGroup(
            self.authz_token, group.id, group.ownerId)

    def _send_users_added_to_group(self, internal_user_ids, group):
        for internal_user_id in internal_user_ids:
            user_id, gateway_id = internal_user_id.rsplit("@", maxsplit=1)
            user_profile = self.request.profile_service['user_profile'].getUserProfileById(
                self.authz_token, user_id, gateway_id)
            signals.user_added_to_group.send(
                sender=self.__class__,
                user=user_profile,
                groups=[group],
                request=self.request)


class ProjectViewSet(APIBackedViewSet):
    serializer_class = serializers.ProjectSerializer
    lookup_field = 'project_id'
    pagination_class = APIResultPagination
    pagination_viewname = 'django_airavata_api:project-list'

    def get_list(self):
        view = self

        class ProjectResultIterator(APIResultIterator):
            def get_results(self, limit=-1, offset=0):
                return view.request.airavata_client.getUserProjects(
                    view.authz_token, view.gateway_id, view.username, limit, offset)

        return ProjectResultIterator()

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getProject(
            self.authz_token, lookup_value)

    def perform_create(self, serializer):
        project = serializer.save(
            owner=self.username,
            gatewayId=self.gateway_id)
        project_id = self.request.airavata_client.createProject(
            self.authz_token, self.gateway_id, project)
        project.projectID = project_id
        self._update_most_recent_project(project_id)

    def perform_update(self, serializer):
        project = serializer.save()
        self.request.airavata_client.updateProject(
            self.authz_token, project.projectID, project)
        self._update_most_recent_project(project.projectID)

    @action(detail=False)
    def list_all(self, request):
        projects = self.request.airavata_client.getUserProjects(
            self.authz_token, self.gateway_id, self.username, -1, 0)
        serializer = serializers.ProjectSerializer(
            projects, many=True, context={'request': request})
        return Response(serializer.data)

    @action(detail=True)
    def experiments(self, request, project_id=None):
        experiments = request.airavata_client.getExperimentsInProject(
            self.authz_token, project_id, -1, 0)
        serializer = serializers.ExperimentSerializer(
            experiments, many=True, context={'request': request})
        return Response(serializer.data)

    def _update_most_recent_project(self, project_id):
        prefs = helpers.WorkspacePreferencesHelper().get(self.request)
        prefs.most_recent_project_id = project_id
        prefs.save()


class ExperimentViewSet(mixins.CreateModelMixin,
                        mixins.RetrieveModelMixin,
                        mixins.UpdateModelMixin,
                        GenericAPIBackedViewSet):
    serializer_class = serializers.ExperimentSerializer
    lookup_field = 'experiment_id'

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getExperiment(
            self.authz_token, lookup_value)

    def perform_create(self, serializer):
        experiment = serializer.save(
            gatewayId=self.gateway_id,
            userName=self.username)
        experiment_id = self.request.airavata_client.createExperiment(
            self.authz_token, self.gateway_id, experiment)
        self._update_workspace_preferences(
            project_id=experiment.projectId,
            group_resource_profile_id=experiment.userConfigurationData.groupResourceProfileId,
            compute_resource_id=experiment.userConfigurationData.computationalResourceScheduling.resourceHostId)
        experiment.experimentId = experiment_id

    def perform_update(self, serializer):
        experiment = serializer.save(
            gatewayId=self.gateway_id,
            userName=self.username)
        self.request.airavata_client.updateExperiment(
            self.authz_token, experiment.experimentId, experiment)
        self._update_workspace_preferences(
            project_id=experiment.projectId,
            group_resource_profile_id=experiment.userConfigurationData.groupResourceProfileId,
            compute_resource_id=experiment.userConfigurationData.computationalResourceScheduling.resourceHostId)

    @action(methods=['post'], detail=True)
    def launch(self, request, experiment_id=None):
        try:
            experiment = request.airavata_client.getExperiment(
                self.authz_token, experiment_id)
            if (experiment.enableEmailNotification):
                experiment.emailAddresses = [request.user.email]
            request.airavata_client.updateExperiment(
                self.authz_token, experiment_id, experiment)
            experiment_util.launch(request, experiment_id)
            return Response({'success': True})
        except Exception as e:
            log.exception(f"Failed to launch experiment {experiment_id}", extra={'request': request})
            return Response({'success': False, 'errorMessage': str(e)})

    @action(methods=['get'], detail=True)
    def jobs(self, request, experiment_id=None):
        jobs = request.airavata_client.getJobDetails(
            self.authz_token, experiment_id)
        serializer = serializers.JobSerializer(
            jobs, many=True, context={'request': request})
        return Response(serializer.data)

    @action(methods=['post'], detail=True)
    def clone(self, request, experiment_id=None):
        cloned_experiment_id = experiment_util.clone(request, experiment_id)
        cloned_experiment = request.airavata_client.getExperiment(
            self.authz_token, cloned_experiment_id)
        serializer = self.serializer_class(
            cloned_experiment, context={'request': request})
        return Response(serializer.data)

    @action(methods=['post'], detail=True)
    def cancel(self, request, experiment_id=None):
        try:
            request.airavata_client.terminateExperiment(
                request.authz_token, experiment_id, self.gateway_id)
            return Response({'success': True})
        except Exception as e:
            log.exception("Cancel action has thrown the following error", extra={'request': request})
            raise e

    @action(methods=['post'], detail=True)
    def fetch_intermediate_outputs(self, request, experiment_id=None):
        if "outputNames" not in request.data:
            return Response(status=status.HTTP_400_BAD_REQUEST)
        try:
            experiment_util.intermediate_output.fetch_intermediate_output(
                request, experiment_id, *request.data["outputNames"])
            return Response({'success': True})
        except Exception as e:
            log.exception("fetchIntermediateOutputs failed with the following error", extra={'request': request})
            raise e

    def _update_workspace_preferences(self, project_id,
                                      group_resource_profile_id,
                                      compute_resource_id):
        prefs = helpers.WorkspacePreferencesHelper().get(self.request)
        prefs.most_recent_project_id = project_id
        prefs.most_recent_group_resource_profile_id = group_resource_profile_id
        prefs.most_recent_compute_resource_id = compute_resource_id
        prefs.save()


class ExperimentSearchViewSet(mixins.ListModelMixin, GenericAPIBackedViewSet):
    serializer_class = serializers.ExperimentSummarySerializer
    pagination_class = APIResultPagination
    pagination_viewname = 'django_airavata_api:experiment-search-list'

    def get_list(self):
        view = self

        filters = {}
        for filter_item in self.request.query_params.items():
            if filter_item[0] in ExperimentSearchFields._NAMES_TO_VALUES:
                # Lookup enum value for this ExperimentSearchFields
                search_field = ExperimentSearchFields._NAMES_TO_VALUES[
                    filter_item[0]]
                filters[search_field] = filter_item[1]

        class ExperimentSearchResultIterator(APIResultIterator):
            def get_results(self, limit=-1, offset=0):
                return view.request.airavata_client.searchExperiments(
                    view.authz_token, view.gateway_id, view.username, filters,
                    limit, offset)

        # Preserve query parameters when moving to next and previous links
        return ExperimentSearchResultIterator(query_params=self.request.query_params.copy())

    def get_instance(self, lookup_value):
        raise NotImplementedError()


class FullExperimentViewSet(mixins.RetrieveModelMixin,
                            GenericAPIBackedViewSet):
    serializer_class = serializers.FullExperimentSerializer
    lookup_field = 'experiment_id'

    def get_instance(self, lookup_value):
        """Get FullExperiment instance with resolved references."""
        # TODO: move loading experiment and references to airavata_sdk?
        experimentModel = self.request.airavata_client.getExperiment(
            self.authz_token, lookup_value)
        outputDataProducts = [
            self.request.airavata_client.getDataProduct(self.authz_token,
                                                        output.value)
            for output in experimentModel.experimentOutputs
            if (output.value and
                output.value.startswith('airavata-dp') and
                output.type in (DataType.URI,
                                DataType.STDOUT,
                                DataType.STDERR))]
        outputDataProducts += [
            self.request.airavata_client.getDataProduct(self.authz_token, dp)
            for output in experimentModel.experimentOutputs
            if (output.value and
                output.type == DataType.URI_COLLECTION)
            for dp in output.value.split(',')
            if output.value.startswith('airavata-dp')]
        appInterfaceId = experimentModel.executionId
        try:
            applicationInterface = self.request.airavata_client \
                .getApplicationInterface(self.authz_token, appInterfaceId)
        except Exception as e:
            log.warning(f"Failed to load app interface: {e}")
            applicationInterface = None
        exp_output_views = output_views.get_output_views(
            self.request, experimentModel, applicationInterface)
        inputDataProducts = [
            self.request.airavata_client.getDataProduct(self.authz_token,
                                                        inp.value)
            for inp in experimentModel.experimentInputs
            if (inp.value and
                inp.value.startswith('airavata-dp') and
                inp.type in (DataType.URI,
                             DataType.STDOUT,
                             DataType.STDERR))]
        inputDataProducts += [
            self.request.airavata_client.getDataProduct(self.authz_token, dp)
            for inp in experimentModel.experimentInputs
            if (inp.value and
                inp.type == DataType.URI_COLLECTION)
            for dp in inp.value.split(',')
            if inp.value.startswith('airavata-dp')]
        applicationModule = None
        try:
            if applicationInterface is not None:
                appModuleId = applicationInterface.applicationModules[0]
                applicationModule = self.request.airavata_client \
                    .getApplicationModule(self.authz_token, appModuleId)
            else:
                log.warning(
                    "Cannot load application model since app interface failed to load")
        except Exception:
            log.exception("Failed to load app interface/module", extra={'request': self.request})

        compute_resource_id = None
        user_conf = experimentModel.userConfigurationData
        if user_conf and user_conf.computationalResourceScheduling:
            comp_res_sched = user_conf.computationalResourceScheduling
            compute_resource_id = comp_res_sched.resourceHostId
        try:
            compute_resource = self.request.airavata_client.getComputeResource(
                self.authz_token, compute_resource_id) \
                if compute_resource_id else None
        except Exception:
            log.exception("Failed to load compute resource for {}".format(
                compute_resource_id), extra={'request': self.request})
            compute_resource = None
        if self.request.airavata_client.userHasAccess(
                self.authz_token,
                experimentModel.projectId,
                ResourcePermissionType.READ):
            project = self.request.airavata_client.getProject(
                self.authz_token, experimentModel.projectId)
        else:
            # User may not have access to project, only experiment
            project = None
        job_details = self.request.airavata_client.getJobDetails(
            self.authz_token, lookup_value)
        full_experiment = serializers.FullExperiment(
            experimentModel,
            project=project,
            outputDataProducts=outputDataProducts,
            inputDataProducts=inputDataProducts,
            applicationModule=applicationModule,
            computeResource=compute_resource,
            jobDetails=job_details,
            outputViews=exp_output_views)
        return full_experiment


class ApplicationModuleViewSet(APIBackedViewSet):
    serializer_class = serializers.ApplicationModuleSerializer
    lookup_field = 'app_module_id'

    def get_list(self):
        return self.request.airavata_client.getAccessibleAppModules(
            self.authz_token, self.gateway_id)

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getApplicationModule(
            self.authz_token, lookup_value)

    def perform_create(self, serializer):
        app_module = serializer.save()
        app_module_id = self.request.airavata_client.registerApplicationModule(
            self.authz_token, self.gateway_id, app_module)
        app_module.appModuleId = app_module_id

    def perform_update(self, serializer):
        app_module = serializer.save()
        self.request.airavata_client.updateApplicationModule(
            self.authz_token, app_module.appModuleId, app_module)

    def perform_destroy(self, instance):
        self.request.airavata_client.deleteApplicationModule(
            self.authz_token, instance.appModuleId)

    @action(detail=True)
    def application_interface(self, request, app_module_id):
        all_app_interfaces = request.airavata_client.getAllApplicationInterfaces(
            self.authz_token, self.gateway_id)
        app_interfaces = []
        for app_interface in all_app_interfaces:
            if not app_interface.applicationModules:
                continue
            if app_module_id in app_interface.applicationModules:
                app_interfaces.append(app_interface)
        if len(app_interfaces) == 1:
            serializer = serializers.ApplicationInterfaceDescriptionSerializer(
                app_interfaces[0], context={'request': request})
            return Response(serializer.data)
        elif len(app_interfaces) > 1:
            log.error(
                "More than one application interface found for module {}: {}"
                .format(app_module_id, app_interfaces), extra={'request': request})
            raise Exception(
                'More than one application interface found for module {}'
                .format(app_module_id)
            )
        else:
            raise Http404("No application interface found for module id {}"
                          .format(app_module_id))

    @action(detail=True)
    def application_deployments(self, request, app_module_id):
        all_deployments = self.request.airavata_client.getAllApplicationDeployments(
            self.authz_token, self.gateway_id)
        app_deployments = [
            dep for dep in all_deployments if dep.appModuleId == app_module_id]
        serializer = serializers.ApplicationDeploymentDescriptionSerializer(
            app_deployments, many=True, context={'request': request})
        return Response(serializer.data)

    @action(methods=['post'], detail=True)
    def favorite(self, request, app_module_id):
        helper = helpers.WorkspacePreferencesHelper()
        workspace_preferences = helper.get(request)
        try:
            application_preferences = (
                workspace_preferences.applicationpreferences_set.get(
                    application_id=app_module_id))
            application_preferences.favorite = True
            application_preferences.save()
        except ObjectDoesNotExist:
            workspace_preferences.applicationpreferences_set.create(
                username=request.user.username,
                application_id=app_module_id,
                favorite=True)

        return HttpResponse(status=204)

    @action(methods=['post'], detail=True)
    def unfavorite(self, request, app_module_id):
        helper = helpers.WorkspacePreferencesHelper()
        workspace_preferences = helper.get(request)
        try:
            application_preferences = (
                workspace_preferences.applicationpreferences_set.get(
                    application_id=app_module_id))
            application_preferences.favorite = False
            application_preferences.save()
        except ObjectDoesNotExist:
            workspace_preferences.applicationpreferences_set.create(
                username=request.user.username,
                application_id=app_module_id,
                favorite=False)

        return HttpResponse(status=204)

    @action(detail=False)
    def list_all(self, request, format=None):
        all_modules = self.request.airavata_client.getAllAppModules(
            self.authz_token, self.gateway_id)
        serializer = self.serializer_class(
            all_modules, many=True, context={'request': request})
        return Response(serializer.data)


class ApplicationInterfaceViewSet(APIBackedViewSet):
    serializer_class = serializers.ApplicationInterfaceDescriptionSerializer
    lookup_field = 'app_interface_id'

    def get_list(self):
        return self.request.airavata_client.getAllApplicationInterfaces(
            self.authz_token, self.gateway_id)

    def get_instance(self, lookup_value):
        try:
            return self.request.airavata_client.getApplicationInterface(
                self.authz_token, lookup_value)
        except Exception:
            # If it failed to load, check to see if it exists at all
            all_interfaces = self.request.airavata_client.getAllApplicationInterfaces(
                self.authz_token, self.gateway_id)
            interface_ids = map(lambda i: i.applicationInterfaceId, all_interfaces)
            if lookup_value not in interface_ids:
                raise Http404("Application interface does not exist")
            else:
                raise  # re-raise

    def perform_create(self, serializer):
        application_interface = serializer.save()
        self._update_input_metadata(application_interface)
        log.debug("application_interface: {}".format(application_interface))
        app_interface_id = self.request.airavata_client.registerApplicationInterface(
            self.authz_token, self.gateway_id, application_interface)
        application_interface.applicationInterfaceId = app_interface_id

    def perform_update(self, serializer):
        application_interface = serializer.save()
        self._update_input_metadata(application_interface)
        self.request.airavata_client.updateApplicationInterface(
            self.authz_token,
            application_interface.applicationInterfaceId,
            application_interface)

    def perform_destroy(self, instance):
        self.request.airavata_client.deleteApplicationInterface(
            self.authz_token, instance.applicationInterfaceId)

    def _update_input_metadata(self, app_interface):
        for app_input in app_interface.applicationInputs:
            if app_input.metaData:
                metadata = json.loads(app_input.metaData)
                # Automatically add {showOptions: {isRequired: true/false}} to
                # toggle isRequired on hidden/shown inputs
                if ("editor" in metadata and
                    "dependencies" in metadata["editor"] and
                        "show" in metadata["editor"]["dependencies"]):
                    if "showOptions" not in metadata["editor"]["dependencies"]:
                        metadata["editor"]["dependencies"]["showOptions"] = {}
                    o = metadata["editor"]["dependencies"]["showOptions"]
                    o["isRequired"] = app_input.isRequired
                    app_input.metaData = json.dumps(metadata)

    @action(detail=True)
    def compute_resources(self, request, app_interface_id):
        compute_resources = request.airavata_client.getAvailableAppInterfaceComputeResources(
            self.authz_token, app_interface_id)
        return Response(compute_resources)


class ApplicationDeploymentViewSet(APIBackedViewSet):
    serializer_class = serializers.ApplicationDeploymentDescriptionSerializer
    lookup_field = 'app_deployment_id'

    def get_list(self):
        app_module_id = self.request.query_params.get('appModuleId', None)
        group_resource_profile_id = self.request.query_params.get(
            'groupResourceProfileId', None)
        if (app_module_id and not group_resource_profile_id)\
                or (not app_module_id and group_resource_profile_id):
            raise ParseError("Query params appModuleId and "
                             "groupResourceProfileId are required together.")
        if app_module_id and group_resource_profile_id:
            return self.request.airavata_client.getApplicationDeploymentsForAppModuleAndGroupResourceProfile(
                self.authz_token, app_module_id, group_resource_profile_id)
        else:
            return self.request.airavata_client.getAccessibleApplicationDeployments(
                self.authz_token, self.gateway_id, ResourcePermissionType.READ)

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getApplicationDeployment(
            self.authz_token, lookup_value)

    def perform_create(self, serializer):
        application_deployment = serializer.save()
        app_deployment_id = self.request.airavata_client.registerApplicationDeployment(
            self.authz_token, self.gateway_id, application_deployment)
        application_deployment.appDeploymentId = app_deployment_id

    def perform_update(self, serializer):
        application_deployment = serializer.save()
        self.request.airavata_client.updateApplicationDeployment(
            self.authz_token, application_deployment.appDeploymentId, application_deployment)

    def perform_destroy(self, instance):
        self.request.airavata_client.deleteApplicationDeployment(
            self.authz_token, instance.appDeploymentId)

    @action(detail=True)
    def queues(self, request, app_deployment_id):
        """Return queues for this deployment with defaults overridden by deployment defaults if they exist"""
        app_deployment = self.request.airavata_client.getApplicationDeployment(
            self.authz_token, app_deployment_id)
        compute_resource = request.airavata_client.getComputeResource(
            request.authz_token, app_deployment.computeHostId)
        # Override defaults with app deployment default queue, if defined
        batch_queues = []
        for batch_queue in compute_resource.batchQueues:
            if app_deployment.defaultQueueName:
                if app_deployment.defaultQueueName == batch_queue.queueName:
                    batch_queue.isDefaultQueue = True
                    batch_queue.defaultNodeCount = app_deployment.defaultNodeCount
                    batch_queue.defaultCPUCount = app_deployment.defaultCPUCount
                    batch_queue.defaultWalltime = app_deployment.defaultWalltime
                else:
                    batch_queue.isDefaultQueue = False
            batch_queues.append(batch_queue)
        serializer = serializers.BatchQueueSerializer(
            batch_queues, many=True, context={'request': request})
        return Response(serializer.data)


class ComputeResourceViewSet(mixins.RetrieveModelMixin,
                             GenericAPIBackedViewSet):
    serializer_class = serializers.ComputeResourceDescriptionSerializer
    lookup_field = 'compute_resource_id'

    def get_instance(self, lookup_value, format=None):
        return self.request.airavata_client.getComputeResource(
            self.authz_token, lookup_value)

    @action(detail=False)
    def all_names(self, request, format=None):
        """Return a map of compute resource names keyed by resource id."""
        return Response(
            request.airavata_client.getAllComputeResourceNames(
                request.authz_token))

    @action(detail=False)
    def all_names_list(self, request, format=None):
        """Return a list of compute resource names keyed by resource id."""
        all_names = request.airavata_client.getAllComputeResourceNames(
            request.authz_token)
        return Response([
            {
                'host_id': host_id,
                'host': host,
                'url': request.build_absolute_uri(
                    reverse('django_airavata_api:compute-resource-detail',
                            args=[host_id]))
            } for host_id, host in all_names.items()
        ])

    @action(detail=True)
    def queues(self, request, compute_resource_id, format=None):
        details = request.airavata_client.getComputeResource(
            request.authz_token, compute_resource_id)
        serializer = self.serializer_class(instance=details,
                                           context={'request': request})
        data = serializer.data
        return Response([queue["queueName"] for queue in data["batchQueues"]])


class LocalJobSubmissionView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        job_submission_id = request.query_params["id"]
        local_job_submission = request.airavata_client.getLocalJobSubmission(
            request.authz_token, job_submission_id)
        return Response(
            thrift_utils.create_serializer(
                LOCALSubmission,
                instance=local_job_submission).data)


class CloudJobSubmissionView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        job_submission_id = request.query_params["id"]
        job_submission = request.airavata_client.getCloudJobSubmission(
            request.authz_token, job_submission_id)
        return Response(
            thrift_utils.create_serializer(
                CloudJobSubmission,
                instance=job_submission).data)


class GlobusJobSubmissionView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        job_submission_id = request.query_params["id"]
        job_submission = request.airavata_client.getClo(
            request.authz_token, job_submission_id)
        return Response(
            thrift_utils.create_serializer(
                GlobusJobSubmission,
                instance=job_submission).data)


class SshJobSubmissionView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        job_submission_id = request.query_params["id"]
        job_submission = request.airavata_client.getSSHJobSubmission(
            request.authz_token, job_submission_id)
        return Response(
            thrift_utils.create_serializer(
                SSHJobSubmission,
                instance=job_submission).data)


class UnicoreJobSubmissionView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        job_submission_id = request.query_params["id"]
        job_submission = request.airavata_client.getUnicoreJobSubmission(
            request.authz_token, job_submission_id)
        return Response(
            thrift_utils.create_serializer(
                UnicoreJobSubmission,
                instance=job_submission).data)


class GridFtpDataMovementView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        data_movement_id = request.query_params["id"]
        data_movement = request.airavata_client.getGridFTPDataMovement(
            request.authz_token, data_movement_id)
        return Response(
            thrift_utils.create_serializer(
                GridFTPDataMovement,
                instance=data_movement).data)


class ScpDataMovementView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        data_movement_id = request.query_params["id"]
        data_movement = request.airavata_client.getSCPDataMovement(
            request.authz_token, data_movement_id)
        return Response(
            thrift_utils.create_serializer(
                SCPDataMovement,
                instance=data_movement).data)


class UnicoreDataMovementView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        data_movement_id = request.query_params["id"]
        data_movement = request.airavata_client.getUnicoreDataMovement(
            request.authz_token, data_movement_id)
        return Response(
            thrift_utils.create_serializer(
                UnicoreDataMovement,
                instance=data_movement).data)


class LocalDataMovementView(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        data_movement_id = request.query_params["id"]
        data_movement = request.airavata_client.getLocalDataMovement(
            request.authz_token, data_movement_id)
        return Response(
            thrift_utils.create_serializer(
                LOCALDataMovement,
                instance=data_movement).data)


class DataProductView(APIView):

    serializer_class = serializers.DataProductSerializer
    permission_classes = [IsAuthenticated, DataProductSharedDirPermission]

    def get(self, request, format=None):
        data_product_uri = request.query_params['product-uri']
        data_product = request.airavata_client.getDataProduct(
            request.authz_token, data_product_uri)
        serializer = self.serializer_class(
            data_product, context={'request': request})
        return Response(serializer.data)

    def put(self, request, format=None):
        data_product_uri = request.query_params['product-uri']
        data_product = request.airavata_client.getDataProduct(
            request.authz_token, data_product_uri)
        if request.data and "fileContentText" in request.data:
            user_storage.update_data_product_content(
                request=request,
                data_product=data_product,
                fileContentText=request.data["fileContentText"])
            return self.get(request=request, format=format)
        else:
            return Response(status=status.HTTP_400_BAD_REQUEST)


@api_view(http_method_names=['POST'])
def upload_input_file(request):
    try:
        input_file = request.FILES['file']
        data_product = user_storage.save_input_file(
            request, input_file, content_type=input_file.content_type)
        serializer = serializers.DataProductSerializer(
            data_product, context={'request': request})
        return JsonResponse({'uploaded': True,
                             'data-product': serializer.data})
    except Exception as e:
        log.error("Failed to upload file", exc_info=True, extra={'request': request})
        resp = JsonResponse({'uploaded': False, 'error': str(e)})
        resp.status_code = 500
        return resp


@api_view(http_method_names=['POST'])
def tus_upload_finish(request):
    uploadURL = request.POST['uploadURL']

    def save_upload(file_path, file_name, file_type):
        with open(file_path, 'rb') as uploaded_file:
            return user_storage.save_input_file(request, uploaded_file,
                                                name=file_name, content_type=file_type)
    try:
        data_product = tus.save_tus_upload(uploadURL, save_upload)
        serializer = serializers.DataProductSerializer(
            data_product, context={'request': request})
        return JsonResponse({'uploaded': True,
                             'data-product': serializer.data})
    except Exception as e:
        return exceptions.generic_json_exception_response(e, status=400)


@gzip_page
@api_view()
def download_file(request):
    # TODO: remove this deprecated view
    warnings.warn("download_file view has moved to SDK", DeprecationWarning)
    # redirect to /sdk/download
    data_product_uri = request.GET.get('data-product-uri', '')
    return redirect(user_storage.get_download_url(request, data_product_uri=data_product_uri))


@api_view(http_method_names=['DELETE'])
@permission_classes([IsAuthenticated, DataProductSharedDirPermission])
def delete_file(request):
    # TODO check that user has write access to this file using sharing API
    data_product_uri = request.GET.get('data-product-uri', '')
    data_product = None
    try:
        data_product = request.airavata_client.getDataProduct(
            request.authz_token, data_product_uri)
    except Exception as e:
        log.warning("Failed to load DataProduct for {}"
                    .format(data_product_uri), exc_info=True)
        raise Http404("data product does not exist") from e
    try:
        if (data_product.gatewayId != settings.GATEWAY_ID or
                data_product.ownerName != request.user.username):
            raise PermissionDenied()
        user_storage.delete(request, data_product)
        return HttpResponse(status=204)
    except ObjectDoesNotExist as e:
        raise Http404(str(e)) from e


class UserProfileViewSet(mixins.RetrieveModelMixin,
                         mixins.ListModelMixin,
                         GenericAPIBackedViewSet):
    serializer_class = serializers.UserProfileSerializer

    def get_list(self):
        user_profile_client = self.request.profile_service['user_profile']
        return user_profile_client.getAllUserProfilesInGateway(
            self.authz_token, self.gateway_id, 0, -1)

    def get_instance(self, lookup_value):
        user_profile_client = self.request.profile_service['user_profile']
        return user_profile_client.getUserProfileById(
            self.authz_token, self.request.user.username, self.gateway_id)


class GroupResourceProfileViewSet(APIBackedViewSet):
    serializer_class = serializers.GroupResourceProfileSerializer
    lookup_field = 'group_resource_profile_id'

    def get_list(self):
        return self.request.airavata_client.getGroupResourceList(
            self.authz_token, self.gateway_id)

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getGroupResourceProfile(
            self.authz_token, lookup_value)

    def perform_create(self, serializer):
        group_resource_profile = serializer.save()
        group_resource_profile.gatewayId = self.gateway_id
        group_resource_profile_id = self.request.airavata_client.createGroupResourceProfile(
            authzToken=self.authz_token, groupResourceProfile=group_resource_profile)
        group_resource_profile.groupResourceProfileId = group_resource_profile_id
        # Update the creationTime field on the group resource profile
        new_group_resource_profile = self.request.airavata_client.getGroupResourceProfile(
            self.authz_token, group_resource_profile_id)
        group_resource_profile.creationTime = new_group_resource_profile.creationTime

    def perform_update(self, serializer):
        grp = serializer.save()
        for removed_compute_resource_preference \
                in grp._removed_compute_resource_preferences:
            self.request.airavata_client.removeGroupComputePrefs(
                self.authz_token,
                removed_compute_resource_preference.computeResourceId,
                removed_compute_resource_preference.groupResourceProfileId)
        for removed_compute_resource_policy \
                in grp._removed_compute_resource_policies:
            self.request.airavata_client.removeGroupComputeResourcePolicy(
                self.authz_token,
                removed_compute_resource_policy.resourcePolicyId)
        for removed_batch_queue_resource_policy \
                in grp._removed_batch_queue_resource_policies:
            self.request.airavata_client.removeGroupBatchQueueResourcePolicy(
                self.authz_token,
                removed_batch_queue_resource_policy.resourcePolicyId)
        self.request.airavata_client.updateGroupResourceProfile(
            self.authz_token, grp)

    def perform_destroy(self, instance):
        self.request.airavata_client.removeGroupResourceProfile(
            self.authz_token, instance.groupResourceProfileId)


class SharedEntityViewSet(mixins.RetrieveModelMixin,
                          mixins.UpdateModelMixin,
                          GenericAPIBackedViewSet):
    serializer_class = serializers.SharedEntitySerializer
    lookup_field = 'entity_id'

    def get_instance(self, lookup_value):
        users = {}
        # Only load *directly* granted permissions since these are the only
        # ones that can be edited
        # Load accessible users in order of permission precedence: users that
        # have WRITE permission should also have READ
        users.update(self._load_directly_accessible_users(
            lookup_value, ResourcePermissionType.READ))
        users.update(self._load_directly_accessible_users(
            lookup_value, ResourcePermissionType.WRITE))
        users.update(self._load_directly_accessible_users(
            lookup_value, ResourcePermissionType.MANAGE_SHARING))
        owner_ids = self._load_directly_accessible_users(
            lookup_value, ResourcePermissionType.OWNER)
        # Assume that there is one and only one DIRECT owner (there may be one
        # or more INDIRECT cascading owners, which would the owners of the
        # ancestor entities, but getAllDirectlyAccessibleUsers does not return
        # indirectly cascading owners)
        owner_id = list(owner_ids.keys())[0]
        # Remove owner from the users list
        del users[owner_id]
        user_list = []
        for user_id in users:
            user_list.append({'user': self._load_user_profile(user_id),
                              'permissionType': users[user_id]})
        groups = {}
        groups.update(self._load_directly_accessible_groups(
            lookup_value, ResourcePermissionType.READ))
        groups.update(self._load_directly_accessible_groups(
            lookup_value, ResourcePermissionType.WRITE))
        groups.update(self._load_directly_accessible_groups(
            lookup_value, ResourcePermissionType.MANAGE_SHARING))
        group_list = []
        for group_id in groups:
            group_list.append({'group': self._load_group(group_id),
                               'permissionType': groups[group_id]})
        return {'entityId': lookup_value,
                'userPermissions': user_list,
                'groupPermissions': group_list,
                'owner': self._load_user_profile(owner_id)}

    def _load_accessible_users(self, entity_id, permission_type):
        users = self.request.airavata_client.getAllAccessibleUsers(
            self.authz_token, entity_id, permission_type)
        return {user_id: permission_type for user_id in users}

    def _load_directly_accessible_users(self, entity_id, permission_type):
        users = self.request.airavata_client.getAllDirectlyAccessibleUsers(
            self.authz_token, entity_id, permission_type)
        return {user_id: permission_type for user_id in users}

    def _load_user_profile(self, user_id):
        user_profile_client = self.request.profile_service['user_profile']
        username = user_id[0:user_id.rindex('@')]
        return user_profile_client.getUserProfileById(self.authz_token,
                                                      username,
                                                      settings.GATEWAY_ID)

    def _load_accessible_groups(self, entity_id, permission_type):
        groups = self.request.airavata_client.getAllAccessibleGroups(
            self.authz_token, entity_id, permission_type)
        return {group_id: permission_type for group_id in groups}

    def _load_directly_accessible_groups(self, entity_id, permission_type):
        groups = self.request.airavata_client.getAllDirectlyAccessibleGroups(
            self.authz_token, entity_id, permission_type)
        return {group_id: permission_type for group_id in groups}

    def _load_group(self, group_id):
        group_manager_client = self.request.profile_service['group_manager']
        return group_manager_client.getGroup(self.authz_token, group_id)

    def perform_update(self, serializer):
        shared_entity = serializer.save()
        entity_id = shared_entity['entityId']
        if len(shared_entity['_user_grant_read_permission']) > 0:
            self._share_with_users(
                entity_id, ResourcePermissionType.READ,
                shared_entity['_user_grant_read_permission'])
        if len(shared_entity['_user_grant_write_permission']) > 0:
            self._share_with_users(
                entity_id, ResourcePermissionType.WRITE,
                shared_entity['_user_grant_write_permission'])
        if len(shared_entity['_user_grant_manage_sharing_permission']) > 0:
            self._share_with_users(
                entity_id, ResourcePermissionType.MANAGE_SHARING,
                shared_entity['_user_grant_manage_sharing_permission'])
        if len(shared_entity['_user_revoke_read_permission']) > 0:
            self._revoke_from_users(
                entity_id, ResourcePermissionType.READ,
                shared_entity['_user_revoke_read_permission'])
        if len(shared_entity['_user_revoke_write_permission']) > 0:
            self._revoke_from_users(
                entity_id, ResourcePermissionType.WRITE,
                shared_entity['_user_revoke_write_permission'])
        if len(shared_entity['_user_revoke_manage_sharing_permission']) > 0:
            self._revoke_from_users(
                entity_id, ResourcePermissionType.MANAGE_SHARING,
                shared_entity['_user_revoke_manage_sharing_permission'])
        if len(shared_entity['_group_grant_read_permission']) > 0:
            self._share_with_groups(
                entity_id, ResourcePermissionType.READ,
                shared_entity['_group_grant_read_permission'])
        if len(shared_entity['_group_grant_write_permission']) > 0:
            self._share_with_groups(
                entity_id, ResourcePermissionType.WRITE,
                shared_entity['_group_grant_write_permission'])
        if len(shared_entity['_group_grant_manage_sharing_permission']) > 0:
            self._share_with_groups(
                entity_id, ResourcePermissionType.MANAGE_SHARING,
                shared_entity['_group_grant_manage_sharing_permission'])
        if len(shared_entity['_group_revoke_read_permission']) > 0:
            self._revoke_from_groups(
                entity_id, ResourcePermissionType.READ,
                shared_entity['_group_revoke_read_permission'])
        if len(shared_entity['_group_revoke_write_permission']) > 0:
            self._revoke_from_groups(
                entity_id, ResourcePermissionType.WRITE,
                shared_entity['_group_revoke_write_permission'])
        if len(shared_entity['_group_revoke_manage_sharing_permission']) > 0:
            self._revoke_from_groups(
                entity_id, ResourcePermissionType.MANAGE_SHARING,
                shared_entity['_group_revoke_manage_sharing_permission'])

    def _share_with_users(self, entity_id, permission_type, user_ids):
        self.request.airavata_client.shareResourceWithUsers(
            self.authz_token, entity_id,
            {user_id: permission_type for user_id in user_ids})

    def _revoke_from_users(self, entity_id, permission_type, user_ids):
        self.request.airavata_client.revokeSharingOfResourceFromUsers(
            self.authz_token, entity_id,
            {user_id: permission_type for user_id in user_ids})

    def _share_with_groups(self, entity_id, permission_type, group_ids):
        self.request.airavata_client.shareResourceWithGroups(
            self.authz_token, entity_id,
            {group_id: permission_type for group_id in group_ids})

    def _revoke_from_groups(self, entity_id, permission_type, group_ids):
        self.request.airavata_client.revokeSharingOfResourceFromGroups(
            self.authz_token, entity_id,
            {group_id: permission_type for group_id in group_ids})

    @action(methods=['put'], detail=True)
    def merge(self, request, entity_id=None):
        # Validate updated sharing settings
        updated = self.get_serializer(data=request.data)
        updated.is_valid(raise_exception=True)
        # Get the existing sharing settings and merge in the updated settings
        existing_instance = self.get_object()
        existing = self.get_serializer(instance=existing_instance)
        merged_data = existing.data
        merged_data['userPermissions'] = existing.data['userPermissions'] + \
            updated.initial_data['userPermissions']
        merged_data['groupPermissions'] = existing.data['groupPermissions'] + \
            updated.initial_data['groupPermissions']
        # Create a merged_serializer from the existing sharing settings and the
        # merged settings. This will calculate all permissions that need to be
        # granted and revoked to go from the exisitng settings to the merged
        # settings.
        merged_serializer = self.get_serializer(
            existing_instance, data=merged_data)
        merged_serializer.is_valid(raise_exception=True)
        self.perform_update(merged_serializer)
        return Response(merged_serializer.data)

    @action(methods=['get'], detail=True)
    def all(self, request, entity_id=None):
        """Load direct plus indirectly (inherited) shared permissions."""
        users = {}
        # Load accessible users in order of permission precedence: users that
        # have WRITE permission should also have READ
        users.update(self._load_accessible_users(
            entity_id, ResourcePermissionType.READ))
        users.update(self._load_accessible_users(
            entity_id, ResourcePermissionType.WRITE))
        users.update(self._load_accessible_users(
            entity_id, ResourcePermissionType.MANAGE_SHARING))
        owner_ids = self._load_accessible_users(
            entity_id, ResourcePermissionType.OWNER)
        # Assume that there is one and only one DIRECT owner (there may be one
        # or more INDIRECT cascading owners, which would the owners of the
        # ancestor entities, but getAllAccessibleUsers does not return
        # indirectly cascading owners)
        owner_id = list(owner_ids.keys())[0]
        # Remove owner from the users list
        del users[owner_id]
        user_list = []
        for user_id in users:
            user_list.append({'user': self._load_user_profile(user_id),
                              'permissionType': users[user_id]})
        groups = {}
        groups.update(self._load_accessible_groups(
            entity_id, ResourcePermissionType.READ))
        groups.update(self._load_accessible_groups(
            entity_id, ResourcePermissionType.WRITE))
        groups.update(self._load_accessible_groups(
            entity_id, ResourcePermissionType.MANAGE_SHARING))
        group_list = []
        for group_id in groups:
            group_list.append({'group': self._load_group(group_id),
                               'permissionType': groups[group_id]})
        shared_entity = {'entityId': entity_id,
                         'userPermissions': user_list,
                         'groupPermissions': group_list,
                         'owner': self._load_user_profile(owner_id)}
        serializer = self.serializer_class(
            shared_entity, context={'request': request})
        return Response(serializer.data)


class CredentialSummaryViewSet(APIBackedViewSet):
    serializer_class = serializers.CredentialSummarySerializer

    def get_list(self):
        ssh_creds = self.request.airavata_client.getAllCredentialSummaries(
            self.authz_token, SummaryType.SSH)
        pwd_creds = self.request.airavata_client.getAllCredentialSummaries(
            self.authz_token, SummaryType.PASSWD)
        return ssh_creds + pwd_creds

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getCredentialSummary(
            self.authz_token, lookup_value)

    @action(detail=False)
    def ssh(self, request):
        summaries = self.request.airavata_client.getAllCredentialSummaries(
            self.authz_token, SummaryType.SSH
        )
        serializer = self.get_serializer(summaries, many=True)
        return Response(serializer.data)

    @action(detail=False)
    def password(self, request):
        summaries = self.request.airavata_client.getAllCredentialSummaries(
            self.authz_token, SummaryType.PASSWD
        )
        serializer = self.get_serializer(summaries, many=True)
        return Response(serializer.data)

    @action(methods=['post'], detail=False)
    def create_ssh(self, request):
        if 'description' not in request.data:
            raise ParseError("'description' is required in request")
        description = request.data.get('description')
        token_id = self.request.airavata_client.generateAndRegisterSSHKeys(
            request.authz_token, description)
        credential_summary = self.request.airavata_client.getCredentialSummary(
            request.authz_token, token_id)
        serializer = self.get_serializer(credential_summary)
        return Response(serializer.data)

    @action(methods=['post'], detail=False)
    def create_password(self, request):
        if ('username' not in request.data or
            'password' not in request.data or
                'description' not in request.data):
            raise ParseError("'username', 'password' and 'description' "
                             "are all required in request")
        username = request.data.get('username')
        password = request.data.get('password')
        description = request.data.get('description')
        token_id = self.request.airavata_client.registerPwdCredential(
            request.authz_token, username, password, description)
        credential_summary = self.request.airavata_client.getCredentialSummary(
            request.authz_token, token_id)
        serializer = self.get_serializer(credential_summary)
        return Response(serializer.data)

    def perform_destroy(self, instance):
        if instance.type == SummaryType.SSH:
            self.request.airavata_client.deleteSSHPubKey(
                self.authz_token, instance.token)
        elif instance.type == SummaryType.PASSWD:
            self.request.airavata_client.deletePWDCredential(
                self.authz_token, instance.token)


class CurrentGatewayResourceProfile(APIView):

    def get(self, request, format=None):
        gateway_resource_profile = \
            request.airavata_client.getGatewayResourceProfile(
                request.authz_token, settings.GATEWAY_ID)
        serializer = serializers.GatewayResourceProfileSerializer(
            gateway_resource_profile, context={'request': request})
        return Response(serializer.data)

    def put(self, request, format=None):
        serializer = serializers.GatewayResourceProfileSerializer(
            data=request.data, context={'request': request})
        if serializer.is_valid():
            gateway_resource_profile = serializer.save()
            request.airavata_client.updateGatewayResourceProfile(
                request.authz_token,
                settings.GATEWAY_ID,
                gateway_resource_profile)
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        else:
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


class ExperimentArchiveView(APIView):

    def get(self, request, experiment_id=None, format=None):
        experiment: ExperimentModel = request.airavata_client.getExperiment(
            request.authz_token, experiment_id)
        result = dict(archived=False, archive_name=None, created_date=None,
                      max_age=settings.GATEWAY_USER_DATA_ARCHIVE_MAX_AGE_DAYS)
        try:
            archive_entry = UserDataArchiveEntry.objects.get(
                entry_path=experiment.userConfigurationData.experimentDataDir,
                user_data_archive__rolled_back=False)
            result["archived"] = True
            result["archive_name"] = archive_entry.user_data_archive.archive_name
            result["created_date"] = archive_entry.user_data_archive.created_date
        except UserDataArchiveEntry.DoesNotExist:
            pass
        return Response(result, status=status.HTTP_200_OK)


class StorageResourceViewSet(mixins.RetrieveModelMixin,
                             GenericAPIBackedViewSet):
    serializer_class = serializers.StorageResourceSerializer
    lookup_field = 'storage_resource_id'

    def get_instance(self, lookup_value, format=None):
        return self.request.airavata_client.getStorageResource(
            self.authz_token, lookup_value)

    @action(detail=False)
    def all_names(self, request, format=None):
        """Return a map of compute resource names keyed by resource id."""
        return Response(
            request.airavata_client.getAllStorageResourceNames(
                request.authz_token))


class StoragePreferenceViewSet(APIBackedViewSet):
    serializer_class = serializers.StoragePreferenceSerializer
    lookup_field = 'storage_resource_id'

    def get_list(self):
        return self.request.airavata_client.getAllGatewayStoragePreferences(
            self.authz_token, settings.GATEWAY_ID)

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getGatewayStoragePreference(
            self.authz_token, settings.GATEWAY_ID, lookup_value)

    def perform_create(self, serializer):
        storage_preference = serializer.save()
        self.request.airavata_client.addGatewayStoragePreference(
            self.authz_token,
            settings.GATEWAY_ID,
            storage_preference.storageResourceId,
            storage_preference)

    def perform_update(self, serializer):
        storage_preference = serializer.save()
        self.request.airavata_client.updateGatewayStoragePreference(
            self.authz_token,
            settings.GATEWAY_ID,
            storage_preference.storageResourceId,
            storage_preference)

    def perform_destroy(self, instance):
        self.request.airavata_client.deleteGatewayStoragePreference(
            self.authz_token, settings.GATEWAY_ID, instance.storageResourceId)


class ParserViewSet(mixins.CreateModelMixin,
                    mixins.RetrieveModelMixin,
                    mixins.UpdateModelMixin,
                    mixins.ListModelMixin,
                    GenericAPIBackedViewSet):
    serializer_class = serializers.ParserSerializer
    lookup_field = 'parser_id'

    def get_list(self):
        return self.request.airavata_client.listAllParsers(
            self.authz_token, settings.GATEWAY_ID)

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getParser(
            self.authz_token, lookup_value, settings.GATEWAY_ID)

    def perform_create(self, serializer):
        parser = serializer.save()
        self.request.airavata_client.saveParser(self.authz_token, parser)

    def perform_update(self, serializer):
        parser = serializer.save()
        self.request.airavata_client.saveParser(self.authz_token, parser)


class UserStoragePathView(APIView):
    serializer_class = serializers.UserStoragePathSerializer
    permission_classes = (IsAuthenticated, UserStorageSharedDirPermission)

    def get(self, request, path="/", format=None):
        # AIRAVATA-3460 Allow passing path as a query parameter instead
        path = request.query_params.get('path', path)
        experiment_id = request.query_params.get('experiment-id')
        return self._create_response(request, path, experiment_id=experiment_id)

    def post(self, request, path="/", format=None):
        path = request.data.get('path', path)
        experiment_id = request.data.get('experiment-id')
        if not user_storage.dir_exists(request, path, experiment_id=experiment_id):
            _, resource_path = user_storage.create_user_dir(request, path, experiment_id=experiment_id)
            # create_user_dir may create the directory with a different name
            # than requested, for example, converting spaces to underscores, so
            # use as the path the path that is returned by create_user_dir
            path = resource_path

        data_product = None
        # Handle direct upload
        if 'file' in request.FILES:
            user_file = request.FILES['file']
            data_product = user_storage.save(
                request, path, user_file, content_type=user_file.content_type,
                experiment_id=experiment_id)
        # Handle a tus upload
        elif 'uploadURL' in request.POST:
            uploadURL = request.POST['uploadURL']

            def save_file(file_path, file_name, file_type):
                with open(file_path, 'rb') as uploaded_file:
                    return user_storage.save(request, path, uploaded_file,
                                             name=file_name, content_type=file_type,
                                             experiment_id=experiment_id)
            data_product = tus.save_tus_upload(uploadURL, save_file)
        return self._create_response(request, path, uploaded=data_product, experiment_id=experiment_id)

    # Accept wither to replace file or to replace file content text.
    def put(self, request, path="/", format=None):
        path = request.POST.get('path', path)
        # Replace the file if the request has a file upload.
        if 'file' in request.FILES:
            self.delete(request=request, path=path, format=format)
            dir_path, file_name = os.path.split(path)
            self.post(request=request, path=dir_path, format=format, file_name=file_name)
        # Replace only the file content if the request body has the `fileContentText`
        elif request.data and "fileContentText" in request.data:
            user_storage.update_file_content(
                request=request,
                path=path,
                fileContentText=request.data["fileContentText"])
        else:
            return Response(status=status.HTTP_400_BAD_REQUEST)

        return self._create_response(request=request, path=path)

    def delete(self, request, path="/", format=None):
        path = request.data.get('path', path)
        experiment_id = request.data.get('experiment-id')
        if user_storage.dir_exists(request, path, experiment_id=experiment_id):
            user_storage.delete_dir(request, path, experiment_id=experiment_id)
        else:
            user_storage.delete_user_file(request, path, experiment_id=experiment_id)

        return Response(status=204)

    def _create_response(self, request, path, uploaded=None, experiment_id=None):
        if user_storage.dir_exists(request, path, experiment_id=experiment_id):
            directories, files = user_storage.listdir(request, path, experiment_id=experiment_id)
            data = {
                'isDir': True,
                'directories': directories,
                'files': files
            }
            if uploaded is not None:
                data['uploaded'] = uploaded
            data['parts'] = self._split_path(path)
            data['path'] = path
            serializer = self.serializer_class(
                data, context={'request': request})
            return Response(serializer.data)
        else:
            file = user_storage.get_file_metadata(request, path, experiment_id=experiment_id)
            data = {
                'isDir': False,
                'directories': [],
                'files': [file]
            }
            if uploaded is not None:
                data['uploaded'] = uploaded
            data['parts'] = self._split_path(path)
            serializer = self.serializer_class(
                data, context={'request': request})
            return Response(serializer.data)

    def _split_path(self, path):
        head, tail = os.path.split(path)
        if head != path:
            return self._split_path(head) + [tail]
        elif tail != "":
            return [tail]
        else:
            return []


class ExperimentStoragePathView(APIView):

    serializer_class = serializers.ExperimentStoragePathSerializer

    def get(self, request, experiment_id=None, path="", format=None):
        return self._create_response(request, experiment_id, path)

    def _create_response(self, request, experiment_id, path):
        if user_storage.experiment_dir_exists(request, experiment_id, path):
            directories, files = user_storage.list_experiment_dir(request, experiment_id, path)

            def add_expid(d):
                d['experiment_id'] = experiment_id
                return d
            data = {
                'isDir': True,
                'directories': map(add_expid, directories),
                'files': map(add_expid, files)
            }
            data['parts'] = self._split_path(path)
            serializer = self.serializer_class(
                data, context={'request': request})
            return Response(serializer.data)
        else:
            raise Http404(f"Path '{path}' does not exist for {experiment_id}")

    def _split_path(self, path):
        head, tail = os.path.split(path)
        if head != "":
            return self._split_path(head) + [tail]
        elif tail != "":
            return [tail]
        else:
            return []


class WorkspacePreferencesView(APIView):
    serializer_class = serializers.WorkspacePreferencesSerializer

    def get(self, request, format=None):
        helper = helpers.WorkspacePreferencesHelper()
        workspace_preferences = helper.get(request)
        serializer = self.serializer_class(
            workspace_preferences, context={'request': request})
        return Response(serializer.data)


class ManageNotificationViewSet(APIBackedViewSet):
    serializer_class = serializers.NotificationSerializer
    lookup_field = 'notification_id'

    def get_instance(self, lookup_value):
        return self.request.airavata_client.getNotification(
            self.authz_token, settings.GATEWAY_ID, lookup_value)

    def get_list(self):
        return self.request.airavata_client.getAllNotifications(
            self.authz_token, self.gateway_id)

    def perform_destroy(self, instance):
        self.request.airavata_client.deleteNotification(
            self.authz_token, settings.GATEWAY_ID, instance.notificationId)

    def perform_create(self, serializer):
        notification = serializer.save(gatewayId=self.gateway_id)
        notificationId = self.request.airavata_client.createNotification(
            self.authz_token, notification)
        notification.notificationId = notificationId

        serializer.update_notification_extension(self.request, notification)

    def perform_update(self, serializer):
        notification = serializer.save()
        self.request.airavata_client.updateNotification(
            self.authz_token, notification)

        serializer.update_notification_extension(self.request, notification)


class AckNotificationViewSet(APIView):

    def get(self, request, format=None):
        if 'id' in request.GET:
            notification_id = request.GET['id']
            try:
                notification = models.User_Notifications.objects.get(
                    notification_id=notification_id,
                    username=request.user.username)
                notification.is_read = True
                notification.save()
            except ObjectDoesNotExist:
                models.User_Notifications.objects.create(
                    username=request.user.username,
                    notification_id=notification.notificationId)
        return HttpResponse(status=204)


class IAMUserViewSet(mixins.RetrieveModelMixin,
                     mixins.UpdateModelMixin,
                     mixins.ListModelMixin,
                     mixins.DestroyModelMixin,
                     GenericAPIBackedViewSet):
    serializer_class = serializers.IAMUserProfile
    pagination_class = APIResultPagination
    permission_classes = (IsAuthenticated, IsInAdminsGroupPermission,)
    lookup_field = 'user_id'

    def get_list(self):
        search = self.request.GET.get('search', None)

        convert_user_profile = self._convert_user_profile

        class IAMUsersResultIterator(APIResultIterator):
            def get_results(self, limit=-1, offset=0):
                return map(convert_user_profile,
                           iam_admin_client.get_users(offset, limit, search))
        return IAMUsersResultIterator(query_params=self.request.query_params.copy())

    def get_instance(self, lookup_value):
        return self._convert_user_profile(
            iam_admin_client.get_user(lookup_value))

    def perform_update(self, serializer):
        managed_user_profile = serializer.save()
        group_manager_client = self.request.profile_service['group_manager']
        user_profile_client = self.request.profile_service['user_profile']
        user_id = managed_user_profile['airavataInternalUserId']
        added_groups = []
        for group_id in managed_user_profile['_added_group_ids']:
            group = group_manager_client.getGroup(self.authz_token, group_id)
            group_manager_client.addUsersToGroup(
                self.authz_token, [user_id], group_id)
            added_groups.append(group)
        if len(added_groups) > 0:
            user_profile = user_profile_client.getUserProfileById(
                self.authz_token,
                managed_user_profile['userId'],
                settings.GATEWAY_ID)
            signals.user_added_to_group.send(
                sender=self.__class__,
                user=user_profile,
                groups=added_groups,
                request=self.request)
        for group_id in managed_user_profile['_removed_group_ids']:
            group_manager_client.removeUsersFromGroup(
                self.authz_token, [user_id], group_id)

    def perform_destroy(self, instance):
        iam_admin_client.delete_user(instance['userId'])

    @action(methods=['post'], detail=True)
    def enable(self, request, user_id=None):
        iam_admin_client.enable_user(user_id)
        instance = self.get_instance(user_id)
        serializer = self.serializer_class(instance=instance,
                                           context={'request': request})
        return Response(serializer.data)

    @action(methods=['put'], detail=False)
    def update_username(self, request):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        old_username = serializer.validated_data['userId']
        new_username = serializer.validated_data['newUsername']
        iam_admin_client.update_username(old_username, new_username)
        # set username_initialized to True so it is treated as valid.
        django_user = get_user_model().objects.get(username=old_username)
        django_user.user_profile.username_initialized = True
        django_user.user_profile.save()
        # Not strictly necessary since next time the user logs in, the Django
        # user record for the user will get updated to have the new username.
        # But this is done to keep it consistent.
        django_user.username = new_username
        django_user.save()
        instance = self.get_instance(new_username)
        serializer = self.serializer_class(instance=instance,
                                           context={'request': request})
        return Response(serializer.data)

    def _convert_user_profile(self, user_profile):
        user_profile_client = self.request.profile_service['user_profile']
        group_manager_client = self.request.profile_service['group_manager']
        airavata_user_profile_exists = user_profile_client.doesUserExist(
            self.authz_token, user_profile.userId, self.gateway_id)
        groups = []
        if airavata_user_profile_exists:
            groups = group_manager_client.getAllGroupsUserBelongs(
                self.authz_token, user_profile.airavataInternalUserId)
        return {
            'airavataInternalUserId': user_profile.airavataInternalUserId,
            'userId': user_profile.userId,
            'gatewayId': user_profile.gatewayId,
            'email': user_profile.emails[0],
            'firstName': user_profile.firstName,
            'lastName': user_profile.lastName,
            'enabled': user_profile.State == Status.ACTIVE,
            'emailVerified': (user_profile.State == Status.CONFIRMED or
                              user_profile.State == Status.ACTIVE),
            'airavataUserProfileExists': airavata_user_profile_exists,
            'creationTime': user_profile.creationTime,
            'groups': groups
        }


class ExperimentStatisticsView(APIView):
    # TODO: restrict to only Admins or Read Only Admins group members
    serializer_class = serializers.ExperimentStatisticsSerializer

    def get(self, request, format=None):
        if 'fromTime' in request.GET:
            from_time = view_utils.convert_utc_iso8601_to_date(
                request.GET['fromTime']).timestamp() * 1000
        else:
            from_time = (datetime.utcnow() -
                         timedelta(days=7)).timestamp() * 1000
        from_time = int(from_time)
        if 'toTime' in request.GET:
            to_time = view_utils.convert_utc_iso8601_to_date(
                request.GET['toTime']).timestamp() * 1000
        else:
            to_time = datetime.utcnow().timestamp() * 1000
        to_time = int(to_time)
        username = request.GET.get('userName', None)
        application_name = request.GET.get('applicationName', None)
        resource_hostname = request.GET.get('resourceHostName', None)
        limit = int(request.GET.get('limit', '50'))
        offset = int(request.GET.get('offset', '0'))

        statistics = request.airavata_client.getExperimentStatistics(
            request.authz_token, settings.GATEWAY_ID, from_time, to_time,
            username, application_name, resource_hostname, limit, offset)
        serializer = self.serializer_class(statistics, context={'request': request})

        paginator = pagination.LimitOffsetPagination()
        paginator.count = statistics.allExperimentCount
        paginator.limit = limit
        paginator.offset = offset
        paginator.request = request
        response = paginator.get_paginated_response(serializer.data)
        # Also add limit and offset to the response
        response.data['limit'] = limit
        response.data['offset'] = offset
        return response


class UnverifiedEmailUserViewSet(mixins.ListModelMixin,
                                 mixins.RetrieveModelMixin,
                                 GenericAPIBackedViewSet):
    serializer_class = serializers.UnverifiedEmailUserProfile
    pagination_class = APIResultPagination
    permission_classes = (IsAuthenticated, IsInAdminsGroupPermission,)
    lookup_field = 'user_id'

    def get_list(self):
        get_users = self._get_unverified_email_user_profiles

        class UnverifiedEmailUsersResultIterator(APIResultIterator):
            def get_results(self, limit=-1, offset=0):
                return get_users(limit, offset)
        return UnverifiedEmailUsersResultIterator()

    def get_instance(self, lookup_value):
        users = self._get_unverified_email_user_profiles(
            limit=1, username=lookup_value)
        if len(users) == 0:
            raise Http404("No unverified email record found for user {}"
                          .format(lookup_value))
        else:
            return users[0]

    def _get_unverified_email_user_profiles(
            self, limit=-1, offset=0, username=None):
        unverified_emails = EmailVerification.objects.filter(
            verified=False).order_by('username').values('username').distinct()
        if username is not None:
            unverified_emails = unverified_emails.filter(username=username)
        if limit > 0:
            unverified_emails = unverified_emails[offset:offset + limit]
        results = []
        for unverified_email in unverified_emails:
            unverified_username = unverified_email['username']
            if iam_admin_client.is_user_exist(unverified_username):
                user_profile = iam_admin_client.get_user(unverified_username)
                if (user_profile.State == Status.CONFIRMED or
                        user_profile.State == Status.ACTIVE):
                    # TODO: test this
                    EmailVerification.objects.filter(
                        username=unverified_username).update(
                        verified=True)
                    continue
                results.append({
                    'userId': user_profile.userId,
                    'gatewayId': user_profile.gatewayId,
                    'email': user_profile.emails[0],
                    'firstName': user_profile.firstName,
                    'lastName': user_profile.lastName,
                    'enabled': user_profile.State == Status.ACTIVE,
                    'emailVerified': (user_profile.State == Status.CONFIRMED or
                                      user_profile.State == Status.ACTIVE),
                    'creationTime': user_profile.creationTime,
                })
            else:
                # Delete the EmailVerification records since that user no
                # longer exists in the IAM service
                EmailVerification.objects.filter(
                    username=unverified_username).delete()
        return results


class LogRecordConsumer(APIView):
    serializer_class = serializers.LogRecordSerializer

    def post(self, request, format=None):
        serializer = self.serializer_class(data=request.data)
        serializer.is_valid(raise_exception=True)
        log_record = serializer.validated_data
        log_level = getattr(logging, log_record['level'], None)
        if log_level is not None:
            stacktrace = "".join(
                map(lambda a: "\n    " + a, log_record['stacktrace']))
            log.log(log_level,
                    "Frontend error: {}: {}\nstacktrace: {}".format(
                        log_record['message'],
                        json.dumps(log_record['details'], indent=4),
                        stacktrace), extra={'request': request})
        return Response(serializer.data)


class SettingsAPIView(APIView):
    serializer_class = serializers.SettingsSerializer

    def get(self, request, format=None):
        data = {
            'fileUploadMaxFileSize': settings.FILE_UPLOAD_MAX_FILE_SIZE,
            'tusEndpoint': settings.TUS_ENDPOINT,
            'pgaUrl': settings.PGA_URL
        }
        serializer = self.serializer_class(
            data, context={'request': request})
        return Response(serializer.data)


class APIServerStatusCheckView(APIView):

    def get(self, request, format=None):
        try:
            request.airavata_client.getUserProjects(request.authz_token,
                                                    settings.GATEWAY_ID,
                                                    request.user.username,
                                                    1,  # limit
                                                    0)  # offset
            data = {
                "apiServerUp": True
            }
        except Exception as e:
            log.debug("API server status check failed: {}".format(str(e)))
            data = {
                "apiServerUp": False
            }
        return Response(data)


@api_view()
def notebook_output_view(request):
    provider_id = request.GET['provider-id']
    experiment_id = request.GET['experiment-id']
    experiment_output_name = request.GET['experiment-output-name']
    data = output_views.generate_data(request,
                                      provider_id,
                                      experiment_output_name,
                                      experiment_id)
    return HttpResponse(data['output'])


@api_view()
def html_output_view(request):
    data = _generate_output_view_data(request)
    return JsonResponse(data)


@api_view()
def image_output_view(request):
    data = _generate_output_view_data(request)
    # data should contain 'image' as a file-like object or raw bytes with the
    # file data and 'mime-type' with the images mimetype
    data['image'] = base64.b64encode(data['image']).decode('utf-8')
    return JsonResponse(data)


@api_view()
def link_output_view(request):
    data = _generate_output_view_data(request)
    return JsonResponse(data)


def _generate_output_view_data(request):
    params = request.GET.copy()
    provider_id = params.pop('provider-id')[0]
    experiment_id = params.pop('experiment-id')[0]
    experiment_output_name = params.pop('experiment-output-name')[0]
    test_mode = ('test-mode' in params and params.pop('test-mode')[0] == "true")
    return output_views.generate_data(request,
                                      provider_id,
                                      experiment_output_name,
                                      experiment_id,
                                      test_mode=test_mode,
                                      **params.dict())


class QueueSettingsCalculatorViewSet(mixins.ListModelMixin, mixins.RetrieveModelMixin, GenericAPIBackedViewSet):
    serializer_class = serializers.QueueSettingsCalculatorSerializer

    def get_list(self):
        return queue_settings_calculators.get_all()

    def get_instance(self, lookup_value):
        calcs = queue_settings_calculators.get_all()
        calc = [calc for calc in calcs if calc.id == lookup_value]
        if len(calc) == 0:
            return None
        return calc[0]

    @action(methods=['post'], detail=True, serializer_class=serializers.ExperimentSerializer)
    def calculate(self, request, pk=None):

        serializer = self.get_serializer(data=request.data)
        result = {}
        # Just ignore invalid experiment model since likely caused by late initialization
        if serializer.is_valid():
            experiment_model = serializer.save()
            result = queue_settings_calculators.calculate_queue_settings(pk, request, experiment_model)
        return Response(result)