libraries/edge/opensearch/granulewriter.py

from types import *
import logging
import urllib.request, urllib.parse, urllib.error
import json

from edge.opensearch.responsewriter import ResponseWriter
from edge.dateutility import DateUtility
from edge.httputility import HttpUtility
from edge.spatialsearch import SpatialSearch
import re

class GranuleWriter(ResponseWriter):
    def __init__(self, configFilePath, requiredParams = None):
        super(GranuleWriter, self).__init__(configFilePath, requiredParams)
        self.startIndex = 0
        self.entriesPerPage = self._configuration.getint('solr', 'entriesPerPage')

    def get(self, requestHandler):
        super(GranuleWriter, self).get(requestHandler)
        #searchParameters = {}
        #logging.debug('uri: '+str(requestHandler.request.headers))
        
        #startIndex = 0
        try:
            self.startIndex = requestHandler.get_argument('startIndex')
        except:
            pass

        #entriesPerPage = self._configuration.getint('solr', 'entriesPerPage')
        try:
            self.entriesPerPage = requestHandler.get_argument('itemsPerPage')
            #cap entries per age at 400
            if (int(self.entriesPerPage) > 400):
                self.entriesPerPage = 400
            self.searchParameters['itemsPerPage'] = self.entriesPerPage
        except:
            pass

        #pretty = True
        try:
            if requestHandler.get_argument('pretty').lower() == 'false':
                self.pretty = False
                self.searchParameters['pretty'] = 'false'
        except:
            pass

        try:
            if requestHandler.get_argument('full').lower() == 'true':
                self.searchParameters['full'] = 'true'
        except:
            pass
        
        try:
            self.searchParameters['format'] = requestHandler.get_argument('format')
        except:
            pass

        parameters = ['startTime', 'endTime', 'keyword', 'granuleName', 'datasetId', 'shortName', 'bbox', 'sortBy']
        #variables = {}
        for parameter in parameters:
            try:
                value = requestHandler.get_argument(parameter)
                self.variables[parameter] = value
                self.searchParameters[parameter] = value
            except:
                pass

        if 'keyword' in self.variables:
            self.variables['keyword'] = self.variables['keyword'].replace('*', '')
            self.variables['keyword'] = self.variables['keyword'].lower()

        #Fetch dataset metadata from Solr
        datasetVariables = {}
        if 'datasetId' in self.variables:
            datasetVariables['datasetId'] = self.variables['datasetId']
        if 'shortName' in self.variables:
            datasetVariables['shortName'] = self.variables['shortName']
        self._getSingleSolrDatasetResponse(datasetVariables, self._onSolrDetermineProcessLevelResponse)

    def _getSolrResponse(self, startIndex, entriesPerPage, variables):
        query = self._constructSolrQuery(startIndex, entriesPerPage, variables)
        url = self._configuration.get('solr', 'granuleUrl')

        httpUtility = HttpUtility()
        httpUtility.getResponse(url+'/select/?'+query, self._onSolrResponse)

    def _constructSolrQuery(self, startIndex, entriesPerPage, variables):
        #set default sort order
        sort='Granule-StartTimeLong+desc'
        filterQuery = None
        queries = []
        for key, value in variables.items():
            #query = ''
            if key == 'startTime':
                startTime = DateUtility.convertISOToUTCTimestamp(value)
                if startTime is not None:
                    query = 'Granule-StopTimeLong:'
                    query += '['+str(startTime)+'%20TO%20*]'
                    queries.append(query)
            elif key == 'endTime':
                stopTime = DateUtility.convertISOToUTCTimestamp(value)
                if stopTime is not None:
                    query = 'Granule-StartTimeLong:'
                    query += '[*%20TO%20'+str(stopTime)+']'
                    queries.append(query)
            elif key == 'keyword':
                newValue = urllib.parse.quote(value)

                query = 'SearchableText-LowerCased:('+newValue+')'
                queries.append(query)
            elif key == 'datasetId':
                query = 'Dataset-PersistentId:'+self._urlEncodeSolrQueryValue(value)
                queries.append(query)
            elif key == 'shortName':
                query = 'Dataset-ShortName-Full:'+self._urlEncodeSolrQueryValue(value)
                queries.append(query)
            elif key == 'granuleName':
                query = 'Granule-Name-Full:'+self._urlEncodeSolrQueryValue(value)
                queries.append(query)
            elif key == 'granuleIds':
                granuleIds = []
                for granuleId in value:
                    granuleIds.append(str(granuleId))
                query = 'Granule-Id:('+'+OR+'.join(granuleIds)+')'
                queries.append(query)

                startIndex = 0
            elif key == 'sortBy':
                sortByMapping = {'timeAsc': 'Granule-StartTimeLong+asc'}
                if value in list(sortByMapping.keys()):
                    sort = sortByMapping[value]
            elif key == 'bbox':
                filterQuery = self._constructBoundingBoxQuery(value)
            #if query != '':
            #    queries.append('%2B'+query)

        if len(queries) == 0:
            queries.append('*')

        query = 'q='+'+AND+'.join(queries)+'&fq=Granule-AccessType:(OPEN+OR+PREVIEW+OR+SIMULATED+OR+REMOTE)+AND+Granule-Status:ONLINE&version=2.2&start='+str(startIndex)+'&rows='+str(entriesPerPage)+'&indent=on&wt=json&sort='+sort
        if filterQuery is not None:
            query += '&' + filterQuery
        logging.debug('solr query: '+query)
        
        return query
    
    def _onSolrDetermineProcessLevelResponse(self, response):
        try:
            #Determine dataset processing level
            processingLevel = None
            solrJson = json.loads(response.body)
            if len(solrJson['response']['docs']) >= 1:
                if 'bbox' in self.variables:
                    processingLevel = solrJson['response']['docs'][0]['Dataset-ProcessingLevel-Full'][0]
                
                    if processingLevel is not None and processingLevel.find('2') != -1:
                        if self._configuration.get('service', 'bbox') == 'l2':
                            #Call Matt's L2 Search Service
                            #raise Exception(self._configuration.get('service', 'l2')+'?'+requestHandler.request.query)
                            httpUtility = HttpUtility()
                            url = self._configuration.get('service', 'l2') + '?'
                            if 'format' not in self.requestHandler.request.arguments:
                                url += 'format=atom&'
                            url += self.requestHandler.request.query
                            logging.debug("Calling L2 Service: " + url)
                            result = httpUtility.getResponse(url, self._onL2Response)
                        else:
                            points = self.variables['bbox'].split(',')
                            if len(points) == 4:
                                spatialSearch = SpatialSearch(
                                    self._configuration.get('service', 'database')
                                )
                                spatialResult = spatialSearch.searchGranules(
                                    int(self.startIndex),
                                    int(self.entriesPerPage),
                                    float(points[0]),
                                    float(points[1]),
                                    float(points[2]),
                                    float(points[3])
                                )
                                logging.debug("Granule spatial search returned")
                                #if len(spatialResult[0]) > 0:
                                self.variables['granuleIds'] = spatialResult[0]
                                self.variables['granuleIdsFound'] = spatialResult[1]
                
                            del self.variables['bbox']
                            solrJson = {'responseHeader': {'params': {}}, 'response': {}}
                            solrJson['response']['numFound'] = int(self.variables['granuleIdsFound'])
                            solrJson['response']['start'] = int(self.startIndex)
                            solrJson['responseHeader']['params']['rows'] = int(self.entriesPerPage)
                            solrJson['response']['docs'] = []
                            for name in self.variables['granuleIds']:
                               solrJson['response']['docs'].append({'Granule-Name': [name]})
                            solrResponse = json.dumps(solrJson)
                            
                            searchText = ''
                            if 'keyword' in self.variables:
                                searchText = self.variables['keyword']
                            openSearchResponse = self._generateOpenSearchResponse(
                                solrResponse,
                                searchText,
                                self._configuration.get('service', 'url')+self.requestHandler.request.path,
                                self.searchParameters,
                                self.pretty
                            )
                            
                            self.requestHandler.set_header("Content-Type", "application/xml")
                            #requestHandler.set_header("Content-Type", "application/rss+xml")
                            #requestHandler.write(solrResponse)
                            self.requestHandler.write(openSearchResponse)
                            self.requestHandler.finish()
                    else:
                        #Dataset is not an L2 dataset so handle search via Solr
                        try:
                            self._getSolrResponse(self.startIndex, self.entriesPerPage, self.variables)
                        except:
                            logging.exception('Failed to get solr response.')
                else:
                    #Not a bounding box search so handle search via Solr
                    try:
                        self._getSolrResponse(self.startIndex, self.entriesPerPage, self.variables)
                    except:
                        logging.exception('Failed to get solr response.')
            else:
                #Dataset metadata cannot be retreived so return empty search result
                solrJson = {'responseHeader': {'params': {}}, 'response': {}}
                solrJson['response']['numFound'] = 0
                solrJson['response']['start'] = int(self.startIndex)
                solrJson['responseHeader']['params']['rows'] = int(self.entriesPerPage)
                solrJson['response']['docs'] = []
                solrResponse = json.dumps(solrJson)
                
                self._writeResponse(solrResponse)
        except BaseException as exception:
            logging.exception('Failed to determine dataset processing level for bbox search ' + str(exception))
            self._handleException(str(exception))

    def _onL2Response(self, response):
        if response.error:
            self._handleException(str(response.error))
        else:
            try:
                logging.debug('header: Content-Type '+response.headers['Content-Type'])
                self.requestHandler.set_header('Content-Type', response.headers['Content-Type'])
                logging.debug('header: Content-Length '+response.headers['Content-Length'])
                self.requestHandler.set_header('Content-Length', response.headers['Content-Length'])
            except:
                pass
            self.requestHandler.write(response.body)
            self.requestHandler.finish()