import datetime import pycurl from io import StringIO import json import logging import urllib.request, urllib.parse, urllib.error import os.path from edge.opensearch.templateresponse import TemplateResponse class SolrCmrTemplateResponse(TemplateResponse): def __init__(self, configuration, link, parameters): super(SolrCmrTemplateResponse, self).__init__() self._configuration = configuration self.link = link self.parameters = parameters def generate(self, solrResponse, pretty=False): self._populate(solrResponse) return super(SolrCmrTemplateResponse, self).generate(pretty) def _get_cmr_response(self, url, cmr): # Execute the curl command and load the json object. buffer = StringIO() try: c = pycurl.Curl() c.setopt(c.URL, url) c.setopt(c.WRITEDATA, buffer) c.perform() c.close() except: output = '{ "errors": "%s" }' %(c.errstr()) c.close() else: try: output = json.loads(buffer.getvalue()) except: output = '{ "errors": "json loads failed: [%s]" }' %(buffer.getvalue()) # Format the output if there are errors. response = {} if list(output.keys()).__contains__('errors'): response['entry'] = [] response['updated'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") response['id'] = url if url.find('collections') != -1: response['title'] = 'ECHO dataset metadata' elif url.find('granules') != -1: response['title'] = 'ECHO granule metadata' response['errors'] = output['errors'][0] else: response = output try: if list(response.keys()).__contains__('errors'): return(response) if not(list(response.keys()).__contains__('feed')): raise ValueError('no "feed" in the cmr response') if not(list(response['feed'].keys()).__contains__('entry')): raise ValueError('no "entry" in the cmr response') if not(list(response['feed'].keys()).__contains__('updated')): raise ValueError('no "updated" key in the cmr response') if not(list(response['feed'].keys()).__contains__('id')): raise ValueError('no "id" key in the cmr response') if not(list(response['feed'].keys()).__contains__('title')): raise ValueError('no "id" key in the cmr response') # Create lists if they do not exists. if not(list(cmr.keys()).__contains__('cmr_search_updated')): cmr['cmr_search_updated'] = [] if not(list(cmr.keys()).__contains__('cmr_search_url')): cmr['cmr_search_url'] = [] if not(list(cmr.keys()).__contains__('cmr_search_title')): cmr['cmr_search_title'] = [] cmr['cmr_search_updated'].append(response['feed']['updated']) cmr['cmr_search_url'].append(response['feed']['id']) cmr['cmr_search_title'].append(response['feed']['title']) # Create list objects for the "entry" key. if response['feed']['entry'] != []: entry = response['feed']['entry'][0] for key in entry: keyname = 'cmr_%s' %(key) if not(list(cmr.keys()).__contains__(keyname)): cmr[keyname] = [] cmr[keyname].append(entry[key]) except ValueError as e: msg = 'Error! parse error: %s.' %e print('%s\n' %msg) return(cmr) def _populate(self, solrResponse): self.variables['link'] = self.link start = 0 rows = 0 numFound = 0 if solrResponse is not None: solrJson = json.loads(solrResponse) logging.debug('Total doc count: '+str(solrJson['response']['numFound'])) logging.debug('Total item count: '+ str(len(solrJson['response']['docs']))) #---------------------------------------------------------------------------------------------- # CMR: Processing #---------------------------------------------------------------------------------------------- cmr_total_time = datetime.timedelta(0) cmr_total_count = 0 for i in range(len(solrJson['response']['docs'])): doc = solrJson['response']['docs'][i] logging.debug('doc[id]: '+str(doc['id'])) #------------------------------------------------------------------------------------------ # CMR: Initialize #------------------------------------------------------------------------------------------ cmr = {} #------------------------------------------------------------------------------------------ # CMR: PRODUCT_TYPE #------------------------------------------------------------------------------------------ if list(solrJson['response']['docs'][i].keys()).__contains__('product_type_dataset_short_name_list'): for j in range(len(doc['product_type_dataset_short_name_list'])): cmr_test = 0 if cmr_test == 1: cmr_search_url = 'https://cmr.earthdata.nasa.gov/search/collections.json?keyword=NSIDC' else: cmr_search_url = 'https://cmr.earthdata.nasa.gov/search/collections.json?keyword=' + \ doc['product_type_dataset_short_name_list'][j] logging.debug('cmr search url: ' +cmr_search_url) time1 = datetime.datetime.now() cmr = self._get_cmr_response(cmr_search_url, cmr) time2 = datetime.datetime.now() time3 = time2 - time1 logging.debug('cmr search time: ' + time3.__str__() + ' 1 product_type') cmr_total_time = cmr_total_time + time3 cmr_total_count = cmr_total_count + 1 #------------------------------------------------------------------------------------------ # CMR: PRODUCT (Only search when the query contains 'id' - ie individual cmr search) #------------------------------------------------------------------------------------------ elif list(solrJson['response']['docs'][i].keys()).__contains__('product_granule_remote_granule_ur_list') and \ list(self.parameters.keys()).__contains__('id'): for j in range(len(doc['product_granule_remote_granule_ur_list'])): cmr_test = 0 if cmr_test == 1: cmr_search_url = 'https://cmr.earthdata.nasa.gov/search/granules.json?granule_ur[]=' + \ '20070106-NAR18_SST-EUR-L2P-sst1nar_noaa18_20070106_asc-v01.nc' else: cmr_search_url = 'https://cmr.earthdata.nasa.gov/search/granules.json?granule_ur[]=' + \ doc['product_granule_remote_granule_ur_list'][j] logging.debug('cmr search url: ' +cmr_search_url) time1 = datetime.datetime.now() cmr = self._get_cmr_response(cmr_search_url, cmr) time2 = datetime.datetime.now() time3 = time2 - time1 logging.debug('cmr search time: ' + time3.__str__() + ' 1 product') cmr_total_time = cmr_total_time + time3 cmr_total_count = cmr_total_count + 1 #------------------------------------------------------------------------------------------ # CMR: Docs not containing CMR search related criteria #------------------------------------------------------------------------------------------ else: continue #------------------------------------------------------------------------------------------ # CMR: Post-processing #------------------------------------------------------------------------------------------ if cmr != {}: doc.update(cmr) #print json.dumps(cmr, indent=4) else: logging.debug('***>>> cmr was not set') #------------------------------------------------------------------------------------------ # CMR: End #------------------------------------------------------------------------------------------ self.variables['docs'] = solrJson['response']['docs'] self.variables['numFound'] = solrJson['response']['numFound'] self.variables['itemsPerPage'] = solrJson['responseHeader']['params']['rows'] self.variables['startIndex'] = solrJson['response']['start'] self.variables['updated'] = datetime.datetime.utcnow().isoformat() + 'Z' start = int(solrJson['response']['start']) rows = int(solrJson['responseHeader']['params']['rows']) numFound = int(solrJson['response']['numFound']) logging.debug('Num docs found: ' + str(self.variables['numFound'])) logging.debug('Items per page: ' + str(self.variables['itemsPerPage'])) # Show CMR statistics. cmr_type = '' if os.path.basename(self.variables['link']) == 'product_type': cmr_type = 'dataset' elif os.path.basename(self.variables['link']) == 'product': cmr_type = 'granule' if (cmr_type == 'dataset' or cmr_type == 'granule') and cmr_total_count != 0: cmr_average_time = cmr_total_time/cmr_total_count cmr_stats_msg = 'CMR search results: %s total per/%d %s(s) %s average per/%s' \ %(cmr_total_time.__str__(), cmr_total_count, cmr_type, cmr_average_time.__str__(), cmr_type) logging.debug(cmr_stats_msg) if 'facet_counts' in solrJson: self.variables['facets'] = solrJson['facet_counts'] self.parameters['startIndex'] = start self.variables['myself'] = self.link + '?' + urllib.parse.urlencode(self.parameters, True) if rows != 0: self.parameters['startIndex'] = numFound - (numFound % rows) self.variables['last'] = self.link + '?' + urllib.parse.urlencode(self.parameters, True) self.parameters['startIndex'] = 0 self.variables['first'] = self.link + '?' + urllib.parse.urlencode(self.parameters, True) if start > 0: if (start - rows > 0): self.parameters['startIndex'] = start - rows self.variables['prev'] = self.link + '?' + urllib.parse.urlencode(self.parameters, True) if start + rows < numFound: self.parameters['startIndex'] = start + rows self.variables['next'] = self.link + '?' + urllib.parse.urlencode(self.parameters, True)