# Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import string from netCDF4 import Dataset, num2date import sys import datetime import csv from collections import OrderedDict import logging #TODO: Get rid of numpy errors? #TODO: Update big SDAP README LOGGER = logging.getLogger("cdms_reader") def assemble_matches(filename): """ Read a CDMS netCDF file and return a list of matches. Parameters ---------- filename : str The CDMS netCDF file name. Returns ------- matches : list List of matches. Each list element is a dictionary. For match m, netCDF group GROUP (PrimaryData or SecondaryData), and group variable VARIABLE: matches[m][GROUP]['matchID']: MatchedRecords dimension ID for the match matches[m][GROUP]['GROUPID']: GROUP dim dimension ID for the record matches[m][GROUP][VARIABLE]: variable value """ try: # Open the netCDF file with Dataset(filename, 'r') as cdms_nc: # Check that the number of groups is consistent w/ the MatchedGroups # dimension assert len(cdms_nc.groups) == cdms_nc.dimensions['MatchedGroups'].size,\ ("Number of groups isn't the same as MatchedGroups dimension.") matches = [] matched_records = cdms_nc.dimensions['MatchedRecords'].size # Loop through the match IDs to assemble matches for match in range(0, matched_records): match_dict = OrderedDict() # Grab the data from each platform (group) in the match for group_num, group in enumerate(cdms_nc.groups): match_dict[group] = OrderedDict() match_dict[group]['matchID'] = match ID = cdms_nc.variables['matchIDs'][match][group_num] match_dict[group][group + 'ID'] = ID for var in cdms_nc.groups[group].variables.keys(): match_dict[group][var] = cdms_nc.groups[group][var][ID] # Create a UTC datetime field from timestamp dt = num2date(match_dict[group]['time'], cdms_nc.groups[group]['time'].units) match_dict[group]['datetime'] = dt LOGGER.info(match_dict) matches.append(match_dict) return matches except (OSError, IOError) as err: LOGGER.exception("Error reading netCDF file " + filename) raise err def assemble_matches_by_primary(filename): """ Read a CDMS netCDF file and return a list of matches, in which secondary data points are grouped together by their primary data point match. This function returns matches in a different order than the 'assemble_matches' function. In this function, all secondary data is associated with its primary match without the need to access multiple matches. Parameters ---------- filename : str The CDMS netCDF file name. Returns ------- matches : list List of matches. Each list element is a dictionary that maps a primary record to all of its associated secondary records. For match m, netCDF group GROUP (PrimaryData or SecondaryData), and group variable VARIABLE: matches[m][GROUP]['matchID']: MatchedRecords dimension ID for the match matches[m][GROUP]['GROUPID']: GROUP dim dimension ID for the record matches[m][GROUP][VARIABLE]: variable value. Each VARIABLE is returned as a masked array. ex. To access the first secondary time value available for a given match: matches[m]['SecondaryData']['time'][0] """ try: # Open the netCDF file with Dataset(filename, 'r') as cdms_nc: # Check that the number of groups is consistent w/ the MatchedGroups # dimension assert len(cdms_nc.groups) == cdms_nc.dimensions['MatchedGroups'].size,\ ("Number of groups isn't the same as MatchedGroups dimension.") matched_records = cdms_nc.dimensions['MatchedRecords'].size primary_matches = cdms_nc.groups['PrimaryData'].dimensions['dim'].size matches = [OrderedDict()] * primary_matches for match in range(matched_records): PID = int(cdms_nc.variables['matchIDs'][match][0]) if len(matches[PID]) == 0: #establishes ordered dictionary for first match[PID] matches[PID] = OrderedDict() for group_num, group in enumerate(cdms_nc.groups): if group_num == 0: #primary if group not in matches[PID].keys(): #initialization matches[PID][group] = OrderedDict() matches[PID][group]['matchID'] = [] matches[PID][group]['matchID'].append(match) ID = cdms_nc.variables['matchIDs'][match][group_num] matches[PID][group][group + 'ID'] = ID for var in cdms_nc.groups[group].variables.keys(): matches[PID][group][var] = cdms_nc.groups[group][var][ID] dt = num2date(matches[PID][group]['time'], cdms_nc.groups[group]['time'].units) matches[PID][group]['datetime'] = dt elif group_num == 1: #secondary if group not in matches[PID].keys(): #initialization matches[PID][group] = OrderedDict() matches[PID][group]['matchID'] = [] matches[PID][group][group + 'ID'] = [] matches[PID][group]['datetime'] = [] matches[PID][group]['matchID'].append(match) ID = cdms_nc.variables['matchIDs'][match][group_num] matches[PID][group][group + 'ID'].append(ID) for var in cdms_nc.groups[group].variables.keys(): if var not in matches[PID][group].keys(): matches[PID][group][var] = [] matches[PID][group][var].append(cdms_nc.groups[group][var][ID]) dt = num2date(matches[PID][group]['time'], cdms_nc.groups[group]['time'].units) matches[PID][group]['datetime'].append(dt[0]) return matches except (OSError, IOError) as err: LOGGER.exception("Error reading netCDF file " + filename) raise err def matches_to_csv(matches, csvfile): """ Write the CDMS matches to a CSV file. Include a header of column names which are based on the group and variable names from the netCDF file. Parameters ---------- matches : list The list of dictionaries containing the CDMS matches as returned from assemble_matches. csvfile : str The name of the CSV output file. """ # Create a header for the CSV. Column names are GROUP_VARIABLE or # GROUP_GROUPID. header = [] for key, value in matches[0].items(): for otherkey in value.keys(): header.append(key + "_" + otherkey) try: # Write the CSV file with open(csvfile, 'w') as output_file: csv_writer = csv.writer(output_file) csv_writer.writerow(header) for match in matches: row = [] for group, data in match.items(): for value in data.values(): row.append(value) csv_writer.writerow(row) except (OSError, IOError) as err: LOGGER.exception("Error writing CSV file " + csvfile) raise err def get_globals(filename): """ Write the CDMS global attributes to a text file. Additionally, within the file there will be a description of where all the different outputs go and how to best utlize this program. Parameters ---------- filename : str The name of the original '.nc' input file. """ x0 = "README / cdms_reader.py Program Use and Description:\n" x1 = "\nThe cdms_reader.py program reads a CDMS netCDF (a NETCDF file with a matchIDs variable)\n" x2 = "file into memory, assembles a list of matches of primary and secondary data\n" x3 = "and optionally\n" x4 = "output the matches to a CSV file. Each matched pair contains one primary\n" x5 = "data record and one secondary data record.\n" x6 = "\nBelow, this file wil list the global attributes of the .nc (NETCDF) file.\n" x7 = "If you wish to see a full dump of the data from the .nc file,\n" x8 = "please utilize the ncdump command from NETCDF (or look at the CSV file).\n" try: with Dataset(filename, "r", format="NETCDF4") as ncFile: txtName = filename.replace(".nc", ".txt") with open(txtName, "w") as txt: txt.write(x0 + x1 +x2 +x3 + x4 + x5 + x6 + x7 + x8) txt.write("\nGlobal Attributes:") for x in ncFile.ncattrs(): txt.write(f'\t :{x} = "{ncFile.getncattr(x)}" ;\n') except (OSError, IOError) as err: LOGGER.exception("Error reading netCDF file " + filename) print("Error reading file!") raise err def create_logs(user_option, logName): """ Write the CDMS log information to a file. Additionally, the user may opt to print this information directly to stdout, or discard it entirely. Parameters ---------- user_option : str The result of the arg.log 's interpretation of what option the user selected. logName : str The name of the log file we wish to write to, assuming the user did not use the -l option. """ if user_option == 'N': print("** Note: No log was created **") elif user_option == '1': #prints the log contents to stdout logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S', handlers=[ logging.StreamHandler(sys.stdout) ]) else: #prints log to a .log file logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S', handlers=[ logging.FileHandler(logName) ]) if user_option != 1 and user_option != 'Y': print(f"** Bad usage of log option. Log will print to {logName} **") if __name__ == '__main__': """ Execution: python cdms_reader.py filename OR python3 cdms_reader.py filename OR python3 cdms_reader.py filename -c -g OR python3 cdms_reader.py filename --csv --meta Note (For Help Try): python3 cdms_reader.py -h OR python3 cdms_reader.py --help """ u0 = '\n%(prog)s -h OR --help \n' u1 = '%(prog)s filename -c -g\n%(prog)s filename --csv --meta\n' u2 ='Use -l OR -l1 to modify destination of logs' p = argparse.ArgumentParser(usage= u0 + u1 + u2) #below block is to customize user options p.add_argument('filename', help='CDMS netCDF file to read') p.add_argument('-c', '--csv', nargs='?', const= 'Y', default='N', help='Use -c or --csv to retrieve CSV output') p.add_argument('-g', '--meta', nargs='?', const='Y', default='N', help='Use -g or --meta to retrieve global attributes / metadata') p.add_argument('-l', '--log', nargs='?', const='N', default='Y', help='Use -l or --log to AVOID creating log files, OR use -l1 to print to stdout/console') #arguments are processed by the next line args = p.parse_args() logName = args.filename.replace(".nc", ".log") create_logs(args.log, logName) cdms_matches = assemble_matches(args.filename) if args.csv == 'Y' : matches_to_csv(cdms_matches, args.filename.replace(".nc",".csv")) if args.meta == 'Y' : get_globals(args.filename)