tools/cdms/cdms_reader.py

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import string
from netCDF4 import Dataset, num2date
import sys
import datetime
import csv
from collections import OrderedDict
import logging


#TODO: Get rid of numpy errors?
#TODO: Update big SDAP README

LOGGER =  logging.getLogger("cdms_reader")

def assemble_matches(filename):
    """
    Read a CDMS netCDF file and return a list of matches.
    
    Parameters
    ----------
    filename : str
        The CDMS netCDF file name.
    
    Returns
    -------
    matches : list
        List of matches. Each list element is a dictionary.
        For match m, netCDF group GROUP (PrimaryData or SecondaryData), and
        group variable VARIABLE:
        matches[m][GROUP]['matchID']: MatchedRecords dimension ID for the match
        matches[m][GROUP]['GROUPID']: GROUP dim dimension ID for the record
        matches[m][GROUP][VARIABLE]: variable value 
    """
   
    try:
        # Open the netCDF file

        with Dataset(filename, 'r') as cdms_nc:
            # Check that the number of groups is consistent w/ the MatchedGroups
            # dimension
            assert len(cdms_nc.groups) == cdms_nc.dimensions['MatchedGroups'].size,\
                ("Number of groups isn't the same as MatchedGroups dimension.")
            
            matches = []
            matched_records = cdms_nc.dimensions['MatchedRecords'].size
           
            # Loop through the match IDs to assemble matches
            for match in range(0, matched_records):
                match_dict = OrderedDict()
                # Grab the data from each platform (group) in the match
                for group_num, group in enumerate(cdms_nc.groups):
                    match_dict[group] = OrderedDict()
                    match_dict[group]['matchID'] = match
                    ID = cdms_nc.variables['matchIDs'][match][group_num]
                    match_dict[group][group + 'ID'] = ID
                    for var in cdms_nc.groups[group].variables.keys():
                        match_dict[group][var] = cdms_nc.groups[group][var][ID]
    
                    # Create a UTC datetime field from timestamp
                    dt = num2date(match_dict[group]['time'],
                                  cdms_nc.groups[group]['time'].units)
                    match_dict[group]['datetime'] = dt
                LOGGER.info(match_dict)
                matches.append(match_dict)
            
            return matches
    except (OSError, IOError) as err:
        LOGGER.exception("Error reading netCDF file " + filename)
        raise err

def assemble_matches_by_primary(filename):
    """
    Read a CDMS netCDF file and return a list of matches, in which secondary data
    points are grouped together by their primary data point match.
   
    This function returns matches in a different order than the 'assemble_matches' function.
    In this function, all secondary data is associated with its primary match without the need
    to access multiple matches. 

    Parameters
    ----------
    filename : str
        The CDMS netCDF file name.
    
    Returns
    -------
    matches : list
        List of matches. Each list element is a dictionary that maps a primary record to all of its associated secondary records.
        For match m, netCDF group GROUP (PrimaryData or SecondaryData), and
        group variable VARIABLE:

        matches[m][GROUP]['matchID']: MatchedRecords dimension ID for the match
        matches[m][GROUP]['GROUPID']: GROUP dim dimension ID for the record
        matches[m][GROUP][VARIABLE]: variable value. Each VARIABLE is returned as a masked array. 

        ex. To access the first secondary time value available for a given match:
            matches[m]['SecondaryData']['time'][0]
    """
   
    try:
        # Open the netCDF file
        with Dataset(filename, 'r') as cdms_nc:
            # Check that the number of groups is consistent w/ the MatchedGroups
            # dimension
            assert len(cdms_nc.groups) == cdms_nc.dimensions['MatchedGroups'].size,\
                ("Number of groups isn't the same as MatchedGroups dimension.")
           
            matched_records = cdms_nc.dimensions['MatchedRecords'].size
            primary_matches = cdms_nc.groups['PrimaryData'].dimensions['dim'].size
            matches = [OrderedDict()] * primary_matches

            for match in range(matched_records):
                PID = int(cdms_nc.variables['matchIDs'][match][0])
        
                if len(matches[PID]) == 0: #establishes ordered dictionary for first match[PID]
                    matches[PID] = OrderedDict()

                for group_num, group in enumerate(cdms_nc.groups):
                    
                    if group_num == 0: #primary
                        
                        if group not in matches[PID].keys(): #initialization
                                matches[PID][group] = OrderedDict()
                                matches[PID][group]['matchID'] = []

                        matches[PID][group]['matchID'].append(match)
                        ID = cdms_nc.variables['matchIDs'][match][group_num]
                        matches[PID][group][group + 'ID'] = ID

                        for var in cdms_nc.groups[group].variables.keys():
                            matches[PID][group][var] = cdms_nc.groups[group][var][ID]
                        
                        dt = num2date(matches[PID][group]['time'], cdms_nc.groups[group]['time'].units)
                        matches[PID][group]['datetime'] = dt

                    elif group_num == 1: #secondary

                        if group not in matches[PID].keys(): #initialization
                            matches[PID][group] = OrderedDict()
                            matches[PID][group]['matchID'] = []
                            matches[PID][group][group + 'ID'] = []
                            matches[PID][group]['datetime'] = []
                        
                        matches[PID][group]['matchID'].append(match)
                        ID = cdms_nc.variables['matchIDs'][match][group_num]
                        matches[PID][group][group + 'ID'].append(ID)
                        
                        for var in cdms_nc.groups[group].variables.keys():
                            if var not in matches[PID][group].keys():
                                matches[PID][group][var] = []
                            matches[PID][group][var].append(cdms_nc.groups[group][var][ID])

                        dt = num2date(matches[PID][group]['time'], cdms_nc.groups[group]['time'].units)
                        matches[PID][group]['datetime'].append(dt[0])
                 
            return matches
    except (OSError, IOError) as err:
        LOGGER.exception("Error reading netCDF file " + filename)
        raise err

def matches_to_csv(matches, csvfile):
    """
    Write the CDMS matches to a CSV file. Include a header of column names
    which are based on the group and variable names from the netCDF file.
    
    Parameters
    ----------
    matches : list
        The list of dictionaries containing the CDMS matches as returned from
        assemble_matches.      
    csvfile : str
        The name of the CSV output file.
    """
    # Create a header for the CSV. Column names are GROUP_VARIABLE or
    # GROUP_GROUPID.
    header = []
    for key, value in matches[0].items():
        for otherkey in value.keys():
            header.append(key + "_" + otherkey)
    
    try:
        # Write the CSV file
        with open(csvfile, 'w') as output_file:
            csv_writer = csv.writer(output_file)
            csv_writer.writerow(header)
            for match in matches:
                row = []
                for group, data in match.items():
                    for value in data.values():
                        row.append(value)
                csv_writer.writerow(row)
    except (OSError, IOError) as err:
        LOGGER.exception("Error writing CSV file " + csvfile)
        raise err

def get_globals(filename):
    """
    Write the CDMS  global attributes to a text file. Additionally,
     within the file there will be a description of where all the different
     outputs go and how to best utlize this program.
    
    Parameters
    ----------      
    filename : str
        The name of the original '.nc' input file.
    
    """
    x0 = "README / cdms_reader.py Program Use and Description:\n"
    x1 = "\nThe cdms_reader.py program reads a CDMS netCDF (a NETCDF file with a matchIDs variable)\n"
    x2 = "file into memory, assembles a list of matches of primary and secondary data\n"
    x3 = "and optionally\n"
    x4 = "output the matches to a CSV file. Each matched pair contains one primary\n"
    x5 = "data record and one secondary data record.\n"
    x6 = "\nBelow, this file wil list the global attributes of the .nc (NETCDF) file.\n"
    x7 = "If you wish to see a full dump of the data from the .nc file,\n"
    x8 = "please utilize the ncdump command from NETCDF (or look at the CSV file).\n"
    try:
        with Dataset(filename, "r", format="NETCDF4") as ncFile:
            txtName = filename.replace(".nc", ".txt")
            with open(txtName, "w") as txt:
                txt.write(x0 + x1 +x2 +x3 + x4 + x5 + x6 + x7 + x8)
                txt.write("\nGlobal Attributes:")
                for x in ncFile.ncattrs():
                    txt.write(f'\t :{x} = "{ncFile.getncattr(x)}" ;\n')


    except (OSError, IOError) as err:
        LOGGER.exception("Error reading netCDF file " + filename)
        print("Error reading file!")
        raise err

def create_logs(user_option, logName):
    """
    Write the CDMS log information to a file. Additionally, the user may
    opt to print this information directly to stdout, or discard it entirely.
    
    Parameters
    ----------      
    user_option : str
        The result of the arg.log 's interpretation of
         what option the user selected.
    logName : str
        The name of the log file we wish to write to,
        assuming the user did not use the -l option.
    """
    if user_option == 'N':
        print("** Note: No log was created **")


    elif user_option == '1':
        #prints the log contents to stdout
        logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
                        level=logging.INFO,
                        datefmt='%Y-%m-%d %H:%M:%S',
                        handlers=[
                            logging.StreamHandler(sys.stdout)
                            ])
                
    else:
        #prints log to a .log file
        logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
                        level=logging.INFO,
                        datefmt='%Y-%m-%d %H:%M:%S',
                        handlers=[
                            logging.FileHandler(logName)
                            ])
        if user_option != 1 and user_option != 'Y':
            print(f"** Bad usage of log option. Log will print to {logName} **")

    


if __name__ == '__main__':
    """
    Execution:
        python cdms_reader.py filename
        OR
        python3 cdms_reader.py filename 
        OR
        python3 cdms_reader.py filename -c -g 
        OR
        python3 cdms_reader.py filename --csv --meta

    Note (For Help Try):
            python3 cdms_reader.py -h
            OR
            python3 cdms_reader.py --help

    """
   
    u0 = '\n%(prog)s -h OR --help \n'
    u1 = '%(prog)s filename -c -g\n%(prog)s filename --csv --meta\n'
    u2 ='Use -l OR -l1 to modify destination of logs'
    p = argparse.ArgumentParser(usage= u0 + u1 + u2)

    #below block is to customize user options
    p.add_argument('filename', help='CDMS netCDF file to read')
    p.add_argument('-c', '--csv', nargs='?', const= 'Y', default='N',
     help='Use -c or --csv to retrieve CSV output')
    p.add_argument('-g', '--meta', nargs='?', const='Y', default='N',
     help='Use -g or --meta to retrieve global attributes / metadata')
    p.add_argument('-l', '--log', nargs='?', const='N', default='Y',
     help='Use -l or --log to AVOID creating log files, OR use -l1 to print to stdout/console') 

    #arguments are processed by the next line
    args = p.parse_args()

    logName = args.filename.replace(".nc", ".log")
    create_logs(args.log, logName)
    
    cdms_matches = assemble_matches(args.filename)

    if args.csv == 'Y' :
        matches_to_csv(cdms_matches, args.filename.replace(".nc",".csv"))

    if args.meta == 'Y' :
        get_globals(args.filename)