gnm_deliverables/views/metadata_views.py

# coding: utf-8

import logging

from django.core.exceptions import ObjectDoesNotExist
from django.db.models.functions import Now
from rest_framework import status
from rest_framework.parsers import JSONParser
from rest_framework.authentication import BasicAuthentication, SessionAuthentication
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.permissions import IsAuthenticated
from gnm_deliverables.inmeta import inmeta_to_string
from django.conf import settings
import os
from gnm_deliverables.jwt_auth_backend import JwtRestAuth
from gnm_deliverables.models import DeliverableAsset, GNMWebsite, Mainstream, Youtube, DailyMotion, \
    LogEntry, Oovvuu
import json
from gnm_deliverables.serializers import *
from rabbitmq.time_funcs import get_current_time
import requests
logger = logging.getLogger(__name__)


class MetadataAPIView(APIView):
    metadata_model = None
    metadata_serializer = None
    authentication_classes = (JwtRestAuth,)

    def get(self, request, project_id, asset_id, *args, **kwargs):
        try:
            metadata = self.metadata_model.objects.get(
                deliverableasset__deliverable__pluto_core_project_id__exact=project_id,
                deliverableasset=asset_id)
            return Response(self.metadata_serializer(metadata).data)
        except ObjectDoesNotExist:
            return Response(status=404)

    def put(self, request, project_id, asset_id, *args, **kwargs):
        try:
            asset = DeliverableAsset.objects.get(
                deliverable__pluto_core_project_id__exact=project_id, pk=asset_id)
            try:
                return self.put_update(request, asset, project_id, asset_id)
            except ObjectDoesNotExist:
                return self.put_insert(request, asset, project_id, asset_id)
        except ObjectDoesNotExist:
            return Response({"status": "error", "detail": "Asset not known"}, status=404)
        except Exception as e:
            return Response({"status": "error", "detail": str(e)}, status=500)

    def put_insert(self, request, asset, project_id, asset_id):
        put = self.metadata_serializer(data=request.data)
        if 'etag' in request.data or self.is_metadata_set(project_id, asset_id):
            return Response({"status": "error", "detail": "conflict"}, status=409)
        if put.is_valid():
            created = put.save()
            self.update_asset_metadata(asset, created)
            asset.save()
            return Response({"status": "ok", "data": self.metadata_serializer(created).data},
                            status=200)
        else:
            return Response({"status": "error", "detail": put.errors}, status=400)

    def put_update(self, request, asset, project_id, asset_id):
        existing = self.metadata_model.objects.get(
            deliverableasset__deliverable__pluto_core_project_id__exact=project_id,
            deliverableasset=asset_id)

        current_etag = existing.etag.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
        if current_etag == request.data.get('etag', None):
            del request.data['etag']
            put = self.metadata_serializer(existing, data=request.data)
            if put.is_valid():
                put.validated_data['etag'] = Now()
                update_count = self.metadata_model.objects.filter(pk=existing.id,
                                                                  etag=current_etag).update(
                    **put.validated_data)
                if update_count == 0:
                    return Response({"status": "error", "detail": "etag conflict"}, status=409)
                elif update_count == 1:
                    updated = self.metadata_model.objects.get(pk=existing.id)
                    return Response(
                        {"status": "ok", "data": self.metadata_serializer(updated).data},
                        status=200)
                else:
                    return Response({"status": "error", "detail": "internal"}, status=500)
            else:
                return Response({"status": "error", "detail": put.errors}, status=400)
        else:
            return Response({"status": "error", "detail": "etag conflict"}, status=409)

    def delete(self, request, project_id, asset_id):
        try:
            entry = self.metadata_model.objects.get(
                deliverableasset__deliverable__pluto_core_project_id__exact=project_id,
                deliverableasset=asset_id)
            entry.delete()
            return Response(status=status.HTTP_204_NO_CONTENT)
        except ObjectDoesNotExist:
            return Response({"status": "error", "detail": "Asset not known"}, status=404)

    def head(self, request, project_id, asset_id):
        try:
            metadata = self.metadata_model.objects.get(
                deliverableasset__deliverable__pluto_core_project_id__exact=project_id,
                deliverableasset=asset_id)
            return Response(status=204, headers={"ETag": metadata.etag})
        except ObjectDoesNotExist:
            return Response({"status": "error", "detail": "asset not found"}, status=404)

    def update_asset_metadata(self, asset, metadata):
        pass

    def is_metadata_set(self, project_id, asset_id):
        pass


class GNMWebsiteAPIView(MetadataAPIView):
    renderer_classes = (JSONRenderer,)
    parser_classes = (JSONParser,)
    metadata_model = GNMWebsite
    metadata_serializer = GNMWebsiteSerializer

    def update_asset_metadata(self, asset, metadata):
        asset.gnm_website_master = metadata

    def is_metadata_set(self, project_id, asset_id):
        asset = DeliverableAsset.objects.get(deliverable__pluto_core_project_id__exact=project_id,
                                             pk=asset_id)
        return asset.gnm_website_master is not None


class MainstreamAPIView(MetadataAPIView):
    renderer_classes = (JSONRenderer,)
    parser_classes = (JSONParser,)
    metadata_model = Mainstream
    metadata_serializer = MainstreamSerializer

    def update_asset_metadata(self, asset, metadata):
        asset.mainstream_master = metadata

    def is_metadata_set(self, project_id, asset_id):
        asset = DeliverableAsset.objects.get(deliverable__pluto_core_project_id__exact=project_id,
                                             pk=asset_id)
        return asset.mainstream_master is not None

    def dispatch(self, request, *args, **kwargs):
        try:
            return super(MainstreamAPIView, self).dispatch(request, *args, **kwargs)
        except Exception as e:
            logger.error("Could not get youtube deliverable metadata: {0}".format(str(e)), e)
            return Response({"status":"error","detail":str(e)}, status=500)


class YoutubeAPIView(MetadataAPIView):
    renderer_classes = (JSONRenderer,)
    parser_classes = (JSONParser,)
    metadata_model = Youtube
    metadata_serializer = YoutubeSerializer

    def update_asset_metadata(self, asset, metadata):
        asset.youtube_master = metadata

    def is_metadata_set(self, project_id, asset_id):
        asset = DeliverableAsset.objects.get(deliverable__pluto_core_project_id__exact=project_id,
                                             pk=asset_id)
        return asset.youtube_master is not None

    def dispatch(self, request, *args, **kwargs):
        try:
            return super(YoutubeAPIView, self).dispatch(request, *args, **kwargs)
        except Exception as e:
            logger.error("Could not get youtube deliverable metadata: {0}".format(str(e)), e)
            return Response({"status":"error","detail":str(e)}, status=500)


class DailyMotionAPIView(MetadataAPIView):
    renderer_classes = (JSONRenderer,)
    parser_classes = (JSONParser,)
    metadata_model = DailyMotion
    metadata_serializer = DailyMotionSerializer

    def update_asset_metadata(self, asset, metadata):
        asset.DailyMotion_master = metadata

    def is_metadata_set(self, project_id, asset_id):
        asset = DeliverableAsset.objects.get(deliverable__pluto_core_project_id__exact=project_id,
                                             pk=asset_id)
        return asset.DailyMotion_master is not None

    def dispatch(self, request, *args, **kwargs):
        try:
            return super(DailyMotionAPIView, self).dispatch(request, *args, **kwargs)
        except Exception as e:
            logger.error("Could not get youtube deliverable metadata: {0}".format(str(e)), e)
            return Response({"status":"error","detail":str(e)}, status=500)


class OovvuuAPIView(MetadataAPIView):
    renderer_classes = (JSONRenderer,)
    parser_classes = (JSONParser,)
    metadata_model = Oovvuu
    metadata_serializer = OovvuuSerializer

    def update_asset_metadata(self, asset, metadata):
        asset.oovvuu_master = metadata

    def is_metadata_set(self, project_id, asset_id):
        asset = DeliverableAsset.objects.get(deliverable__pluto_core_project_id__exact=project_id,
                                             pk=asset_id)
        return asset.oovvuu_master is not None

    def dispatch(self, request, *args, **kwargs):
        try:
            return super(OovvuuAPIView, self).dispatch(request, *args, **kwargs)
        except Exception as e:
            logger.error("Could not get Oovvuu deliverable metadata: {0}".format(str(e)), e)
            return Response({"status":"error","detail":str(e)}, status=500)


class ReutersConnectAPIView(MetadataAPIView):
    renderer_classes = (JSONRenderer,)
    parser_classes = (JSONParser,)
    metadata_model = ReutersConnect
    metadata_serializer = ReutersConnectSerializer

    def update_asset_metadata(self, asset, metadata):
        asset.reutersconnect_master = metadata

    def is_metadata_set(self, project_id, asset_id):
        asset = DeliverableAsset.objects.get(deliverable__pluto_core_project_id__exact=project_id,
                                             pk=asset_id)
        return asset.reutersconnect_master is not None

    def dispatch(self, request, *args, **kwargs):
        try:
            return super(ReutersConnectAPIView, self).dispatch(request, *args, **kwargs)
        except Exception as e:
            logger.error("Could not get Reuters Connect deliverable metadata: {0}".format(str(e)), e)
            return Response({"status": "error", "detail": str(e)}, status=500)


class PlatformLogsView(APIView):
    authentication_classes = (JwtRestAuth,)

    def get(self, request, project_id, asset_id, platform):
        try:
            asset = DeliverableAsset.objects.get(
                deliverable__pluto_core_project_id__exact=project_id, pk=asset_id)
        except ObjectDoesNotExist:
            return Response({"status": "error", "details": "not found"}, status=404)
        if platform == 'youtube' and asset.youtube_master_id:
            log_entries = LogEntry.objects.filter(related_youtube=asset.youtube_master_id)
        elif platform == 'gnmwebsite' and asset.gnm_website_master_id:
            log_entries = LogEntry.objects.filter(
                related_gnm_website_id=asset.gnm_website_master_id)
        elif platform == 'mainstream' and asset.mainstream_master_id:
            log_entries = LogEntry.objects.filter(related_mainstream=asset.mainstream_master_id)
        elif platform == 'dailymotion' and asset.DailyMotion_master_id:
            log_entries = LogEntry.objects.filter(related_daily_motion=asset.DailyMotion_master_id)
        else:
            return Response({"status": "error", "details": "not found"}, status=404)

        qs = log_entries.order_by('-timestamp')
        if "limit" in request.GET:
            try:
                limit = int(request.GET["limit"])
                qs = log_entries.order_by('-timestamp')[0:limit]
            except Exception as e:
                logger.warning("limit parameter {0} was not valid: {1}".format(request.GET['limit'], str(e)))

        if "full" in request.GET:
            data = [LogEntrySerializer(entry).data for entry in qs]
        else:
            data = [entry.log_line for entry in qs]
        return Response({"logs": data}, status=200)


class PlatformLogUpdateView(APIView):
    authentication_classes = (BasicAuthentication, )
    parser_classes = (JSONParser, )
    renderer_classes = (JSONRenderer, )
    permission_classes = (IsAuthenticated, )

    def post(self, request, project_id, asset_id, platform:str):
        """
        receives a log update and saves it.
        expects a JSON body in the format { "sender": "sender-name",
         "log": "log-line",
         "completed":true/false/absent,
         "failed":true/false/absent,
         "uploadedUrl":"url" [optional, ignored unless completed=true and failed=false/absent]
         }.
        logs are timestamped as they arrive
        :param request:
        :param project_id:
        :param asset_id:
        :param platform:
        :return:
        """
        try:
            asset = DeliverableAsset.objects.get(pk=asset_id)

            newentry = LogEntry(
                timestamp=get_current_time(),
                sender=request.data["sender"],
                log_line=request.data["log"]
            )

            did_fail = False
            did_succeed = False
            asset_needs_save = False
            logger.debug("Received CDS update: {0}".format(request.data))

            if "completed" in request.data and request.data["completed"]:
                if "failed" in request.data and request.data["failed"]:
                    did_fail = True
                else:
                    did_succeed = True

            lcplatform = platform.lower()
            if lcplatform=="dailymotion":
                if asset.DailyMotion_master is None:
                    logger.error("Received daily motion syndication progress for {0} but no DN record exists on this asset".format(asset_id))
                    return Response({"status": "invalid_target","detail":"no daily motion metadata to update"}, status=404)
                related_id = asset.DailyMotion_master_id
                newentry.related_daily_motion = asset.DailyMotion_master
                if not did_fail and not did_succeed:
                    if asset.DailyMotion_master.upload_status!='Uploading':
                        asset.DailyMotion_master.upload_status = 'Uploading'
                        asset.DailyMotion_master.save()
                elif did_fail:
                    asset.DailyMotion_master.upload_status='Upload Failed'
                    asset.DailyMotion_master.save()
                elif did_succeed:
                    asset.DailyMotion_master.upload_status='Upload Complete'
                    asset.DailyMotion_master.publication_date = get_current_time()
                    if "uploadedUrl" in request.data:
                        asset.DailyMotion_master.daily_motion_url = request.data["uploadedUrl"]
                    asset.DailyMotion_master.save()
            elif lcplatform=="mainstream":
                if asset.mainstream_master is None:
                    logger.error("Received Mainstream syndication progress for {0} but no MS record exists on this asset".format(asset_id))
                    return Response({"status": "invalid_target","detail":"no MS metadata to update"}, status=404)
                related_id = asset.mainstream_master_id
                newentry.related_mainstream = asset.mainstream_master
                if not did_fail and not did_succeed:
                    if asset.mainstream_master.upload_status!='Uploading':
                        asset.mainstream_master.upload_status = 'Uploading'
                        asset.mainstream_master.save()
                elif did_fail:
                    asset.mainstream_master.upload_status='Upload Failed'
                    asset.mainstream_master.save()
                elif did_succeed:
                    asset.mainstream_master.upload_status='Upload Complete'
                    asset.mainstream_master.publication_date = get_current_time()
                    asset.mainstream_master.save()
            else:
                return Response({"status":"bad_request","detail":"platform not recognised or does not support log entries"}, status=400)

            if related_id is None:
                return Response({"status":"bad_request","detail":"no syndication data for this platform on this id"}, status=400)
            else:
                newentry.save()
                return Response({"status":"ok"},status=200)
        except KeyError as e:
            logger.error("Invalid log updated for {0} {1}: missing key {2}".format(platform, asset_id, str(e)))
            return Response({"status":"bad_request","detail":"{0}: field missing".format(e)},status=400)

        except DeliverableAsset.DoesNotExist:
            return Response({"status":"notfound","detail":"no deliverable asset matching id"},status=404)


class TriggerOutputView(APIView):
    authentication_classes = (JwtRestAuth, )

    def post(self, *args, **kwargs):
        try:
            return self.do_post(*args, **kwargs)
        except Exception as e:
            logger.error("Could not trigger metadata output: {0}".format(str(e)))
            logger.exception("Trace was ", exc_info=e)
            return Response({"status":"uncaught_error", "detail":str(e)})

    def make_sent_note(self, platform:str, user:str, asset:DeliverableAsset):
        """
        updates the syndication notes to mark when this was sent and by whom
        :param platform: platform that the video is sent to
        :param user: username that initiated the send
        :param asset: DeliverableAsset instance that is being sent
        :return: None. Also guaranteed not to raise exceptions; exceptions are caught and logged here.
        """
        try:
            new_record = SyndicationNotes(
                content="Requested send to {0}".format(platform),
                deliverable_asset=asset,
                username=user
            )
            new_record.save()
        except Exception as e:
            logger.error("Could not update syndication notes for send of {0} to {1} by {2}: {3}".format(asset.filename, platform, user, e))

    def do_post(self, request, project_id:int, platform:str, asset_id:int):
        from gnm_deliverables.signals import MessageRelay
        try:
            asset:DeliverableAsset = DeliverableAsset.objects.get(pk=asset_id)
        except DeliverableAsset.DoesNotExist:
            return Response({"status":"error","details":"Asset not found"}, status=404)

        try:
            relay = MessageRelay()
        except Exception as e:
            logger.error("Could not initialise message relay: {0}".format(e))
            return Response({"status":"error","details": "could not initialise message relay"}, status=500)

        routes_map:dict = getattr(settings, "CDS_ROUTE_MAP")
        if routes_map is None:
            logger.error("Could not find CDS_ROUTE_MAP in the configuration")
            return Response({"status":"config_error","details": "CDS_ROUTE_MAP not in configuration"}, status=500)

        platform_name = platform.lower()
        route_name = routes_map.get(platform_name)
        if route_name is None:
            logger.error("Platform {0} was not recognised. CDS_ROUTE_MAP has {1}".format(platform_name, routes_map))
            return Response({"status":"bad_request","detail": "Platform name not recognised"}, status=400)

        routing_key = "deliverables.syndication.{0}.upload".format(platform_name)

        message_content = {
            "inmeta": inmeta_to_string(asset, platform_name),
            "deliverable_asset": asset.pk,
            "deliverable_bundle": asset.deliverable.pk,
            "filename": os.path.basename(asset.filename),
            "online_id": asset.online_item_id,
            "nearline_id": asset.nearline_item_id,
            "archive_id": asset.archive_item_id,
            "routename": route_name
        }
        try:
            encoded_payload = json.dumps(message_content).encode("UTF-8")
        except Exception as e:
            logger.error("Could not encode the message content payload: {0}".format(str(e)))
            logger.error("Offending content was {0}".format(message_content))
            return Response({"status":"error","detail":str(e)})

        try:
            relay.send_content(routing_key, encoded_payload)
            self.make_sent_note(platform, request.user.username, asset)
            return Response({"status":"ok","routing_key":routing_key})
        except Exception as e:
            logger.error("Could not send message to {0}: {1}".format(routing_key, str(e)))
            return Response({"status":"error","detail": "Could not send message to broker"}, status=500)


class ResyncToPublished(APIView):
    authentication_classes = (JwtRestAuth, BasicAuthentication, SessionAuthentication, )    #SessionAuthentication is needed for tests to work
    renderer_classes = (JSONRenderer, )
    permission_classes = (IsAuthenticated, )

    def post(self, request, project_id:int, asset_id:int):
        try:
            asset = DeliverableAsset.objects.get(pk=asset_id)
        except DeliverableAsset.DoesNotExist:
            return Response({"status":"error","details":"Asset not found"}, status=404)

        if asset.atom_id is None:
            return Response({"status":"error","details":"Asset is not from an atom","asset_id":asset_id}, status=400)

        try:
            url = settings.LAUNCH_DETECTOR_URL + "/update/" + str(asset.atom_id)
            logger.info("Update URL for asset {aid} on project {pid} is {url}".format(aid=asset_id,pid=project_id,url=url))
            response = requests.put(url)

            logger.info("Updating {pid}/{aid}: Launch detector said {status} {msg}".format(aid=asset_id,pid=project_id,
                                                                                           status=response.status_code,
                                                                                           msg=response.content))
            #simply echo the Launch Detector's response back to the client
            return Response(response.json(), status=response.status_code)
        except requests.ConnectTimeout:
            return Response({"status": "error", "error": "Timeout connecting to LaunchDetector, please try again and notify multimediatech@theguardian.com"},status=500)
        except requests.ConnectionError:
            return Response({"status": "error", "error": "Unable to connect to LaunchDetector, please notify multimediatech@theguardian.com"},status=500)
        except Exception as e:
            return Response({"status": "error", "error": str(e)},status=500)