gnm_deliverables/models.py

# coding: utf-8

import errno
import logging
import os
import time
import urllib.error
import urllib.parse
import urllib.request
from django.conf import settings
from django.contrib.postgres.fields import ArrayField
from django.db import models
from django.db.models import Q
from django.utils.functional import cached_property
from django.utils.timezone import now
from gnmvidispine.vs_item import VSItem, VSException, VSNotFound
from gnmvidispine.vs_job import VSJob
from .choices import DELIVERABLE_ASSET_STATUS_INGEST_FAILED, \
    DELIVERABLE_ASSET_TYPES_DICT, UPLOAD_STATUS, PRODUCTION_OFFICE, PRIMARY_TONE, \
    PUBLICATION_STATUS, DELIVERABLE_ASSET_STATUS_TRANSCODING
from .choices import DELIVERABLE_ASSET_STATUS_NOT_INGESTED, DELIVERABLE_ASSET_STATUSES, DELIVERABLE_ASSET_STATUS_TRANSCODED
from .choices import DELIVERABLE_ASSET_TYPE_CHOICES, DELIVERABLE_STATUS_ALL_FILES_INGESTED, \
    DELIVERABLE_STATUS_FILES_TO_INGEST, DELIVERABLE_ASSET_TYPE_OTHER_SUBTITLE, \
    DELIVERABLE_ASSET_TYPE_OTHER_TRAILER, \
    DELIVERABLE_ASSET_TYPE_OTHER_MISCELLANEOUS, DELIVERABLE_ASSET_TYPE_OTHER_PAC_FORMS, DELIVERABLE_ASSET_TYPE_OTHER_POST_PRODUCTION_SCRIPT
from .exceptions import ImportFailedError, NoShapeError
from .files import get_path_for_deliverable, find_files_for_deliverable, create_folder, \
    get_local_path_for_deliverable, create_folder_for_deliverable
from .templatetags.deliverable_tags import sizeof_fmt
from .transcodepreset import TranscodePresetFinder
import datetime
logger = logging.getLogger(__name__)


def seconds_to_timestamp(seconds):
    return time.strftime('%H:%M:%S', time.gmtime(float(seconds)))


transcode_preset_finder = TranscodePresetFinder()


class Deliverable(models.Model):
    project_id = models.CharField(null=True, blank=True, max_length=61)
    commission_id = models.BigIntegerField(null=False, blank=False, db_index=True)
    pluto_core_project_id = models.BigIntegerField(null=False, blank=False, db_index=True, unique=True)
    name = models.CharField(null=False, blank=False, unique=True, max_length=255)
    created = models.DateTimeField(null=False, blank=False, auto_now_add=True)

    def sync_assets_from_file_system(self):
        """
        performs a scan of the drop-folder associated with this deliverable. If a file is found that does not correspond
        to a DeliverableAsset record then create a record for it, if there is a corresponding record then update the ctime/
        mtime/atime.
        Also, if any asset records have NOT been imported (i.e. their type and item_id is null) and their corresponding
        files have been removed then delete those records.
        :return: a dictionary with two keys, a count of "added" records and a count of "removed" records.
        """
        assets_on_fs = []
        added_count = 0
        removed_count = 0

        for f in find_files_for_deliverable(self.name):
            asset, created = DeliverableAsset.objects.get_or_create(
                filename=f.path,
                deliverable=self,
                size=f.size,
                modified_dt=f.modified_dt,
                defaults=dict(
                    access_dt=f.access_dt,
                    changed_dt=f.changed_dt,
                    absolute_path=f.absolute_path
                )
            )
            assets_on_fs.append(asset)
            if created:
                logger.info('Asset created: %s' % asset)
                added_count += 1
            else:
                logger.info('Asset already existed: %s' % asset)
                # Update defaults
                asset.access_dt = f.access_dt
                asset.changed_dt = f.changed_dt
                asset.absolute_path = f.absolute_path
                asset.save()

        # Remove assets rows that are not found on the FS and does not have an item tied to it
        assets_to_delete = self.assets.filter(
            type__isnull=True,
            online_item_id__isnull=True
        ).exclude(
            id__in=[a.id for a in assets_on_fs]
        )
        delete_count = assets_to_delete.count()
        if delete_count > 0:
            assets_to_delete.delete()
            removed_count += 1
            logger.info('Deleted %s asset rows' % delete_count)
        return {"added": added_count, "removed": removed_count}

    @cached_property
    def path(self):
        return get_path_for_deliverable(self.name)

    @cached_property
    def local_path(self):
        return get_local_path_for_deliverable(self.name)

    @cached_property
    def local_open_uri(self):
        local_path = self.local_path.rstrip('/') + '/'
        return 'pluto:openfolder:%s' % urllib.parse.quote(local_path)

    def status(self):
        if self.assets.filter(ingest_complete_dt__isnull=True).exists():
            return DELIVERABLE_STATUS_FILES_TO_INGEST
        else:
            return DELIVERABLE_STATUS_ALL_FILES_INGESTED

    def asset_type(self, type):
        assets = self.assets.filter(type=type)
        return None if assets.count() <= 0 else assets[0]

    def create_folder(self):
        return create_folder(self.path)

    def create_folder_for_deliverable(self):
        return create_folder_for_deliverable(self.name)

    def create_asset_from_vs_item(self, item_id, user):
        """
        tries to "adopt" the given existing vidispine item onto this deliverable bundle.
        if the item is already associated with this deliverable bundle then it is returned without re-import.
        can raise NoShapeError if either the item has no original shape, or VSNotFound if the item does not exist.
        can also raise other VSException from gnmvidispine if server communication fails.
        :param item_id: item id to adopt
        :param user: name of the user carrying out the operation, it will be done as this user in VS
        :return: a tuple of (DeliverableAsset, Boolean) where the boolean value indicates whether a new DeliverableAsset
        was created or not
        """
        created = False
        try:
            asset = DeliverableAsset.objects.get(online_item_id=item_id, deliverable=self)
            logger.info(
                'Asset for item "{item_id}" already existed: {asset}'.format(item_id=item_id,
                                                                             asset=asset))
        except DeliverableAsset.DoesNotExist:
            asset = DeliverableAsset(
                online_item_id=item_id,
                deliverable=self
            )
            item = asset.item(user)
            try:
                shape = item.get_shape("master")
            except VSNotFound:
                ##we can tidy the re-raise up later
                raise NoShapeError('No original shape attached to item')

            asset.filename = item.get("originalFilename", allowArray=False)
            asset.absolute_path = item.get("originalUri", allowArray=False)

            files = [x for x in shape.files()]
            if len(files) == 0:
                raise NoShapeError("Original shape had no files on it")
            asset.size = files[0].size
            mtime = None  ##FIXME: sort this out
            asset.ingest_complete_dt = now()
            asset.created_from_existing_item = True
            asset.save()
            created = True

            logger.info(
                'Asset for item "{item_id}" created: {asset}'.format(item_id=item_id, asset=asset))
        return asset, created

    @classmethod
    def for_project(cls, project_id):
        return Deliverable.objects.filter(project_id=project_id)

    @classmethod
    def sync_project(cls, project_id):
        deliverables = cls.for_project(project_id)
        for deliverable in deliverables:
            deliverable.sync_assets_from_file_system()
        return deliverables

    def __str__(self):
        return self.__unicode__()

    def __unicode__(self):
        return '{name}'.format(name=self.name)

    class Meta:
        pass


class DeliverableAsset(models.Model):
    type = models.PositiveIntegerField(null=True, blank=True,
                                       choices=DELIVERABLE_ASSET_TYPE_CHOICES)

    # Stat info
    filename = models.TextField(null=True, blank=True)
    absolute_path = models.TextField(null=True, blank=True)
    size = models.BigIntegerField(null=True, blank=True)
    access_dt = models.DateTimeField(null=True, blank=True)
    modified_dt = models.DateTimeField(null=True, blank=True)
    changed_dt = models.DateTimeField(null=True, blank=True)

    # Fields set on ingest
    job_id = models.TextField(null=True, blank=True)
    online_item_id = models.TextField(null=True, blank=True)
    nearline_item_id = models.TextField(null=True, blank=True)
    archive_item_id = models.TextField(null=True, blank=True)

    ingest_complete_dt = models.DateTimeField(null=True, blank=True)
    file_removed_dt = models.DateTimeField(null=True, blank=True)

    duration_seconds = models.FloatField(null=True, blank=True)
    version = models.IntegerField(null=True, blank=True)
    linked_to_lowres = models.BooleanField(null=True, blank=True)
    
    # Fields set by atom responder
    atom_id = models.UUIDField(null=True)

    # Lifecycle
    status = models.IntegerField(null=False,
                                 choices=DELIVERABLE_ASSET_STATUSES,
                                 default=DELIVERABLE_ASSET_STATUS_NOT_INGESTED)

    created_from_existing_item = models.BooleanField(default=False)

    deliverable = models.ForeignKey(Deliverable, related_name='assets', on_delete=models.PROTECT)

    # Syndication metadata
    gnm_website_master = models.ForeignKey('GNMWebsite', on_delete=models.SET_NULL, null=True)
    youtube_master = models.ForeignKey('Youtube', on_delete=models.SET_NULL, null=True)
    DailyMotion_master = models.ForeignKey('DailyMotion', on_delete=models.SET_NULL, null=True)
    mainstream_master = models.ForeignKey('Mainstream', on_delete=models.SET_NULL, null=True)
    oovvuu_master = models.ForeignKey("Oovvuu", on_delete=models.SET_NULL, null=True)
    reutersconnect_master = models.ForeignKey("ReutersConnect", on_delete=models.SET_NULL, null=True)

    def __init__(self, *args, **kwargs):
        super(DeliverableAsset, self).__init__(*args, **kwargs)
        self.__item = None
        self.__job = None

    def update_metadata(self, user):
        """
        updates the metadata on all storage layers
        :param user:
        :return:
        """
        raise Exception("update_metadata is not implemented yet")
        # set_item_metadata(user, self.item_id, metadata_document_from_dict(dict(title=self.get_name()), md_group='Deliverable'))

    def get_name(self):
        # naming convention
        return '{type} for {name}'.format(type=self.type_string, name=self.deliverable.name)

    def create_placeholder(self, user, commit=True):
        """
        requests Vidispine to create a placeholder item for this deliverable, if it does not have an ID on it.
        if the deliverable does already have an item id, this is a no-op.
        this is automatically called from start_file_import, so you don't need to call it directly.
        :param user: (string) user to run the request as
        :param commit: if True, save the deliverable to the database immediately (default true)
        :return: None
        """
        if not isinstance(user, str):
            raise TypeError("create_placeholder user must be a string, got a {0}".format(user.__class__.__name__))

        if self.online_item_id is None:
            new_item = VSItem(url=settings.VIDISPINE_URL,
                              user=settings.VIDISPINE_USER,
                              passwd=settings.VIDISPINE_PASSWORD,
                              run_as=user)

            builder = new_item.get_metadata_builder()
            builder.addMeta({
                "title": self.get_name()
            })
            builder.addGroup("Asset",{
                "gnm_category": "Deliverable",
                "original_filename": self.absolute_path,
                "gnm_owner": user,
                "gnm_containing_projects": str(self.deliverable.pluto_core_project_id),
                "gnm_file_created": self.modified_dt
            })
            builder.addGroup("Deliverable", {
                "gnm_deliverable_bundle": str(self.deliverable.pluto_core_project_id),
                "gnm_deliverable_id": str(self.id)
            })
            new_item.createPlaceholder(builder.as_xml().decode("utf8"))

            if commit:
                self.save()
            return new_item
            #we don't store stuff in collections any more
            #add_to_collection(user, self.deliverable.project_id, self.item_id)

    def start_file_import(self, user, commit=True):
        """
        tells VS to start importing the file.
         If no item currently exists, then a placeholder is created and the media imported to that.
         If an item does currently exist, then the media is added as a new version.
        This can raise any subclass of gnmvidispine.VSException
        :param user: username to run the operation as
        :param commit: whether to save the model before returning
        :return:
        """
        if self.created_from_existing_item:
            return
        if self.online_item_id is None:
            current_item = self.create_placeholder(user, commit=False)
        else:
            current_item = self.item(user=user)

        should_we_not_attempt_a_transcode = self.type in [DELIVERABLE_ASSET_TYPE_OTHER_MISCELLANEOUS,
                                                          DELIVERABLE_ASSET_TYPE_OTHER_PAC_FORMS,
                                                          DELIVERABLE_ASSET_TYPE_OTHER_POST_PRODUCTION_SCRIPT,
                                                          DELIVERABLE_ASSET_TYPE_OTHER_SUBTITLE]

        #run_as should be taken care of by the VSItem base class, initated from self.item
        import_job = current_item.import_to_shape(uri="file://" + self.absolute_path.replace(" ","%20"),
                                                  priority="MEDIUM",
                                                  essence=True,
                                                  thumbnails=False,
                                                  noTranscode=should_we_not_attempt_a_transcode,
                                                  jobMetadata={
                                                      "import_source": "pluto-deliverables",
                                                      "project_id": str(self.deliverable.pluto_core_project_id),
                                                      "asset_id": str(self.id)
                                                  })
        self.job_id = import_job.name
        if commit:
            self.save()

    def create_proxy(self, priority='MEDIUM'):
        """
        tells VS to start proxying the file.  This is done seperately to the ingest portion, so that we can use
        Vidispine's understanding of the file format in order to determine which is the appropriate transcoding
        to use.
        Can raise either a VSException or a vidispine_api.HttpError if the operation fails.
        This is normally called from the callback endpoint, so we have no user present suitable for "run-as"
        :return:
        """
        item_info = self.item(None)     # caller should catch exceptions from this
        if item_info is None:
            raise ValueError("Item is not imported")

        mime_type = item_info.get("mimeType")
        if mime_type is None:
            logger.error("{0} for deliverable asset {1} in {2}: Could not determine MIME type for transcode".format(item_info.name, self.id, self.deliverable.id))
            return None

        preset_name = transcode_preset_finder.match(mime_type)
        if preset_name is None:
            logger.error("{0} for deliverable asset {1} in {2}: Did not recognise MIME type {3}".format(item_info.name, self.id, self.deliverable.id, mime_type))
            raise ValueError("Did not recognise MIME type of item")

        try:
            # raises VSNotFound if the shape does not exist, which is what we expect.
            item_info.get_shape(preset_name)
            logger.info("Item {0} already has a shape {1}, not transcoding it".format(item_info.name, preset_name))
            self.status = DELIVERABLE_ASSET_STATUS_TRANSCODED
            self.save()
            return  #don't proceed to make a transcode if one already exists!
        except VSNotFound:
            pass

        logger.info("Requesting transcode to {0} of {1}".format(item_info.name, preset_name))
        job_id = item_info.transcode(preset_name, priority, wait=False, create_thumbnails=True,
                                     job_metadata={
                                        "import_source": "pluto-deliverables",
                                        "project_id": str(self.deliverable.pluto_core_project_id),
                                        "asset_id": str(self.id)
                                    })

        self.job_id = job_id
        self.status = DELIVERABLE_ASSET_STATUS_TRANSCODING
        self.save()
        return job_id

    def has_ongoing_job(self, user):
        """
        returns a boolean indicating whether this deliverable has a job ongoing or not
        :param user:
        :return:
        """
        try:
            current_job = self.job(user)
            if current_job is None:
                return None
            else:
                return not current_job.finished()

        except VSNotFound:
            # no job => no ongoing job!
            return None

    @cached_property
    def type_string(self):
        return DELIVERABLE_ASSET_TYPES_DICT.get(self.type)

    @cached_property
    def size_string(self):
        return sizeof_fmt(self.size)

    def type_allows_many(self):
        return self.type == DELIVERABLE_ASSET_TYPE_OTHER_SUBTITLE \
               or self.type == DELIVERABLE_ASSET_TYPE_OTHER_TRAILER \
               or self.type == DELIVERABLE_ASSET_TYPE_OTHER_MISCELLANEOUS

    def _set_cached_item(self, newvalue):
        """
        method for testing, to manually set the cached item. you should not call this directly, but rely on it to be setr
        by calling item()
        :param newvalue:
        :return:
        """
        self.__item = newvalue

    def item(self, user):
        """
        returns a gnmvidispine VSItem object representing the vidispine item associated with this deliverable.
        the first time it is called (this happens internally) the data is lifted from the VS server and thereafter
        it is cached
        the item_id must be set for this to work.  None is returned if the item id is not set; VSNotFound is raised
        if the item does not exist, or another VSException is raised if server communication fails
        :param user: username to run the metadata get as
        :return: the VSItem, possibly from in-memory cache
        """
        if self.__item is not None:
            return self.__item
        if self.online_item_id is not None:
            self.__item = VSItem(url=settings.VIDISPINE_URL, user=settings.VIDISPINE_USER,
                                 passwd=settings.VIDISPINE_PASSWORD, run_as=user)
            self.__item.populate(self.online_item_id)
            return self.__item
        return None

    def job(self, user):
        """
        returns a gnmvidispine VSJob that represents the import job assocaited with this deliverable, or None if there was
        none set.
        can raise a VSNotFound if the job does not exist or another VSException if server communication fails
        :param user:
        :return:
        """
        if self.__job is not None:
            return self.__job
        if self.job_id is not None:
            self.__job = VSJob(url=settings.VIDISPINE_URL, user=settings.VIDISPINE_USER,
                               passwd=settings.VIDISPINE_PASSWORD, run_as=user)
            return self.__job.populate(self.job_id)
        return None

    def purge(self, user=None):
        if self.online_item_id is not None:
            try:
                exists_in_other_deliverable = DeliverableAsset.objects.filter(
                    Q(online_item_id=self.online_item_id) & ~Q(id=self.id)
                ).exists()
                if not self.created_from_existing_item and not exists_in_other_deliverable:
                    self.item(user).delete()

            except VSException:
                logger.exception(
                    'Failed to delete item id "{item_id}" for deliverable asset "{id}"'.format(
                        item_id=self.online_item_id,
                        id=self.id
                    ))
            super(DeliverableAsset, self).delete()

    def remove_file(self):
        """
        Try to remove the original source file from disk, once it's been ingested. Sets file_removed_dt and saves the model.
        :return: True if the file was removed, False if not.
        """
        try:
            os.remove(str(self.absolute_path))
            logger.info('Removed file for asset "{asset}": "{path}"'.format(asset=self,
                                                                            path=self.absolute_path))
            self.file_removed_dt = now()
            self.save()
            return True
        except OSError as e:
            # this _should_ get picked up by the periodic task remove_stale_files
            if e.errno == errno.ENOENT:
                # No such file
                if self.file_removed_dt is None:
                    self.file_removed_dt = now()
                    self.save()
                return False
            else:
                logger.exception('Failed to remove "{path}"'.format(path=self.absolute_path))
                return False

    def __str__(self):
        return '{name}'.format(name=self.filename)


class GNMWebsite(models.Model):
    media_atom_id = models.UUIDField(null=True, blank=True)
    upload_status = models.TextField(null=True, blank=True,
                                     choices=UPLOAD_STATUS, db_index=True)
    production_office = models.TextField(null=True, blank=True, choices=PRODUCTION_OFFICE,
                                         db_index=True)
    tags = ArrayField(models.CharField(null=True, max_length=255), null=True, blank=True,
                      db_index=True)
    publication_date = models.DateTimeField(null=True, blank=True)
    website_title = models.TextField(null=True, blank=True, db_index=True)
    website_description = models.TextField(null=True, blank=True)
    primary_tone = models.TextField(null=True, blank=True, choices=PRIMARY_TONE, db_index=True)
    publication_status = models.TextField(null=True, blank=True, choices=PUBLICATION_STATUS)
    etag = models.DateTimeField(null=False, blank=False, auto_now_add=True)
    source = models.TextField(null=True, blank=True)


class Mainstream(models.Model):
    mainstream_title = models.TextField(null=False, blank=False)
    mainstream_description = models.TextField(null=True, blank=True)
    mainstream_tags = ArrayField(models.CharField(null=False, max_length=255), null=True, blank=True)
    mainstream_rules_contains_adult_content = models.BooleanField()
    upload_status = models.TextField(null=True, blank=True, choices=UPLOAD_STATUS, db_index=True)
    publication_date = models.DateTimeField(null=True, blank=True)
    etag = models.DateTimeField(null=False, blank=False, auto_now_add=True)
    routename = models.TextField(null=True, blank=True)
    job_id = models.TextField(null=True, blank=True)


class Youtube(models.Model):
    youtube_id = models.TextField(null=False, blank=False, db_index=True)
    youtube_title = models.TextField(null=False, blank=False)
    youtube_description = models.TextField(null=True, blank=True)
    youtube_tags = ArrayField(models.CharField(null=True, max_length=255), null=True, blank=True)
    youtube_category = models.CharField(null=False, max_length=255)
    youtube_channel = models.CharField(null=False, max_length=255)
    publication_date = models.DateTimeField(null=True, blank=True)
    etag = models.DateTimeField(null=False, blank=False, auto_now_add=True)

    def __str__(self):
        return '{title}'.format(title=self.youtube_title)


class DailyMotion(models.Model):
    daily_motion_url = models.TextField(null=True, blank=True)
    daily_motion_title = models.TextField(null=False, blank=False)
    daily_motion_description = models.TextField(null=True, blank=True)
    daily_motion_tags = ArrayField(models.CharField(null=True, max_length=255), null=True, blank=True)
    daily_motion_category = models.TextField(default="news")
    publication_date = models.DateTimeField(null=True, blank=True)
    upload_status = models.TextField(null=True, blank=True, choices=UPLOAD_STATUS, db_index=True)
    daily_motion_no_mobile_access = models.BooleanField()
    daily_motion_contains_adult_content = models.BooleanField()
    etag = models.DateTimeField(null=False, blank=False, auto_now_add=True)
    routename = models.TextField(null=True, blank=True)
    job_id = models.TextField(null=True, blank=True)


class Oovvuu(models.Model):
    seen_on_channel = models.BooleanField(default=False)
    etag = models.DateTimeField(null=False, blank=False, auto_now_add=True)


class ReutersConnect(models.Model):
    seen_on_channel = models.BooleanField(default=False)
    etag = models.DateTimeField(null=False, blank=False, auto_now_add=True)


class LogEntry(models.Model):
    timestamp = models.DateTimeField(null=False, blank=False)
    related_gnm_website = models.ForeignKey(GNMWebsite, on_delete=models.CASCADE, null=True,
                                            db_index=True)
    related_youtube = models.ForeignKey(Youtube, on_delete=models.CASCADE, null=True,
                                        db_index=True)
    related_daily_motion = models.ForeignKey(DailyMotion, on_delete=models.CASCADE, null=True,
                                             db_index=True)
    related_mainstream = models.ForeignKey(Mainstream, on_delete=models.CASCADE, null=True,
                                           db_index=True)
    sender = models.TextField(null=False, blank=False, db_index=True)
    log_line = models.TextField(null=False, blank=False)


class SyndicationNotes(models.Model):
    timestamp = models.DateTimeField(default=datetime.datetime.now)
    username = models.TextField(max_length=255)
    content = models.TextField(max_length=32768)
    deliverable_asset = models.ForeignKey(DeliverableAsset, on_delete=models.CASCADE, db_index=True)


class YouTubeCategories(models.Model):
    title = models.CharField(max_length=1024)
    identity = models.CharField(max_length=64, blank=True, null=True)


class YouTubeChannels(models.Model):
    title = models.CharField(max_length=1024)
    identity = models.CharField(max_length=128, blank=True, null=True)