jobs/eam-integrations/scripts/workday_everfi_integration.py

from workday_everfi.api.Workday import WorkdayAPI
from workday_everfi.api.Everfi import EverfiAPI
from api.util import Util, APIAdaptorException
import argparse
import logging
import sys

def cal_user_location(wd_user, locs, loc_map_table):
    loc = ""
    location_country = wd_user.get("location_country", "")

    if location_country == "Canada":
        loc = loc_map_table.get(wd_user.get("location_province", ""), "")
        if not loc:
            loc = "Federal (Canada)"
    elif location_country == "United States of America":
        # if wd_user.get("location_state", "") == "New York":
        loc = loc_map_table.get(wd_user.get("location_state", ""), "")

        if not loc:
            loc = "United States"
    else:
        loc = "Default"

    id = locs.get(loc)["id"]
    if not id:
        id = locs.get("Default")["Id"]

    logger.debug(
        f"Location id={id} mapped for user {wd_user.get('primary_work_email','')} loc = {loc}"
    )
    return id


class Everfi:
    def __init__(self) -> None:
        self.everfi_api = EverfiAPI()
        self.logger = logging.getLogger(self.__class__.__name__)
    
    def get_everfi_users(self, locs, loc_map_table, hire_dates):
        filter = {'filter[active]': 'true'}
        fields = {'fields[users]': 'email,first_name,last_name,sso_id,employee_id,student_id,location_id,active,user_rule_set_roles,category_labels'}
        return self.everfi_api.get_users(fields, filter, locs, loc_map_table, hire_dates)

    def deactivate_users(self, del_list, everfi_users):
        count = 0
        
        for email in del_list:
            id = everfi_users[email].get('id')
            self.everfi_api.deactivate_user(id)
            if '@' in email:
                n = email.split("@")[0]
            else:
                n = email
            self.logger.info(f"{n[:4]} .. {n[-1]} deleted")
            count += 1
            if count % 20 == 0:
                self.logger.info(f"[{count} of {len(del_list)}] users deactivated.")
        return count
        
    def activate_user(self, id):
        self.everfi_api.set_active(id,True)
        
    def get_locations_mapping_table(self):
        return self.everfi_api.get_locations_mapping_table()

    def upd_everfi_users(
        self,
        hire_date_category_id,
        hire_dates,
        locs,
        upd_list_keys,
        wd_users,
        everfi_users,
        loc_map_table,
    ):
        errors_list = []
        count_upd = 0
        loc_id_dict = {x.get('id'):x.get('attributes').get('name') for x in locs.values()}
        
        for email in upd_list_keys:
            wd_user = wd_users[email][1]
            loc_id = cal_user_location(wd_user, locs, loc_map_table)    
            if int(loc_id) != everfi_users[email].get('attributes').get('location_id'):
                if '@' in email:
                    n = email.split("@")[0]
                else:
                    n = email
                self.logger.info(f"User {n[:4]} .. {n[-1]} changed location from {loc_id_dict[str(everfi_users[email].get('attributes').get('location_id'))]} to {loc_id_dict[loc_id]}")
            json_data = {
                "data": {
                    "type": "registration_sets",
                    "id": everfi_users[email]["id"],
                    "attributes": {
                        "registrations": [
                            {
                                "rule_set": "user_rule_set",
                                "first_name": wd_user.get("preferred_first_name", ""),
                                "last_name": wd_user.get("preferred_last_name", ""),
                                "location_id": loc_id,
                                "employee_id": wd_user.get("employee_id", ""),
                                "sso_id": email,
                            },
                            {
                                "rule_set": "cc_learner",
                                "role": "supervisor"
                                if wd_user.get("is_manager", "")
                                else "non_supervisor",
                            },
                        ],
                    },
                },
            }
            try:
                r = self.everfi_api.upd_user(everfi_users[email]["id"], json_data)
            except Exception as e:
                self.logger.exception(e)
                errors_list.append(e)
            
            cat_label_user_id = self.get_category_label_user_id(everfi_users[email]["id"])
            if cat_label_user_id:
                self.delete_category_label_user(cat_label_user_id)

            #wd_users[email][1]["hire_date"] = '2024-07-10'
            hire_date_id = self.get_hire_date_id(
                wd_users[email][1]["hire_date"], hire_date_category_id, hire_dates
            )

            try:
                r = self.everfi_api.assign_label_user(
                    r.data.get("data").get("id"), hire_date_id
                )
            except Exception as e:
                self.logger.exception(e)
                errors_list.append(e)
            
            if count_upd % 20 == 0:
                self.logger.info(f"[{count_upd} of {len(upd_list_keys)}] users updated.")
                
            count_upd += 1
    
        return count_upd
    
    def get_category_label_user_id(self, id):
        ret = self.everfi_api.get_category_label_user_id(id)   
        if len(ret.data.get('data',''))>0:
            return ret.data.get('data','')[0].get('id','')
        else:
            return None
    
    def delete_category_label_user(self, id):
        ret = self.everfi_api.delete_category_label_user(id)               
        return ret
    
    def bulk_clear_category_id(self, ids, category_id,category_label):
        return self.everfi_api.bulk_clear_category_id(ids, category_id,category_label)

    def get_hire_date_id(self, wd_hire_date, hire_date_category_id, hire_dates):
        wd_hire_date = wd_hire_date.split("-")
        wd_hire_date = wd_hire_date[1] + "-" + wd_hire_date[0]
        hire_date_id = hire_dates.get(wd_hire_date)
        if not hire_date_id:
            # add new hire date
            r = self.everfi_api.add_hire_date(
                name=wd_hire_date, category_id=hire_date_category_id
            )
            id = r.data.get("data").get("id")
            hire_dates[wd_hire_date] = id
            return r.data.get("data").get("id")
        return hire_date_id

    def add_everfi_users(
        self,
        hire_date_category_id,
        hire_dates,
        locs,
        add_list_keys,
        wd_users,
        loc_map_table,
    ):
        errors = []
        count_add = 0
        for email in add_list_keys:
            wd_user = wd_users[email][1]
            loc_id = cal_user_location(wd_user, locs, loc_map_table)
            json_data = {
                "data": {
                    "type": "registration_sets",
                    "attributes": {
                        "registrations": [
                            {
                                "rule_set": "user_rule_set",
                                "first_name": wd_user.get("preferred_first_name", ""),
                                "last_name": wd_user.get("preferred_last_name", ""),
                                "email": wd_user.get("primary_work_email", ""),
                                "sso_id": email,
                                "employee_id": wd_user.get("employee_id", ""),
                                "location_id": loc_id,
                            },
                            {
                                "rule_set": "cc_learner",
                                "role": "supervisor"
                                if wd_user.get("is_manager", "")
                                else "non_supervisor",
                            },
                        ],
                    },
                },
            }
            try:
                r = self.everfi_api.add_user(json_data)
            except Exception as e:
                self.logger.exception(e)
                self.logger.info("Trying to activate user and update ")
                if (e.args[0][0].get('id','')=='user_rule_set'):
                    # try to active user
                    # find user by email and then update the user with current data
                    filter = {'filter[email]': wd_user.get("primary_work_email", "")}
                    fields = {'fields[users]': 'id,email'}                    
                    #find user id
                    user = self.everfi_api.search_user(fields, filter)
                    id = user.get(email,'').get('id', '')
                    if id:
                        #self.activate_user(id)
                        json_data['data']['id'] = id
                        json_data['data']['attributes']['registrations'][0]['active'] = True 
                        #active user and update fields
                        r = self.everfi_api.upd_user(id, json_data) 
                        #remove hire date custom field
                       
                        #hd = wd_users[email][1]["hire_date"].split('-') 
                        cat_label_user_id = self.get_category_label_user_id(id)
                        if cat_label_user_id:
                            self.delete_category_label_user(cat_label_user_id)
                        #self.bulk_clear_category_id([id], hire_date_category_id, hd[1] + '-' + hd[0])
                    else:                        
                        errors.append(e)
                        continue

            
            #wd_users[email][1]["hire_date"] = '2024-07-10'
            hire_date_id = self.get_hire_date_id(
                wd_users[email][1]["hire_date"], hire_date_category_id, hire_dates
            )

            try:
                r = self.everfi_api.assign_label_user(
                    r.data.get("data").get("id"), hire_date_id
                )
            except Exception as e:
                self.logger.exception(e)
                errors.append(e)

            count_add += 1
            
            if '@' in email:
                n = email.split("@")[0]
            else:
                n = email
            self.logger.info(f"{n[:4]} .. {n[-1]} added")
            
            if count_add % 20 == 0:
                self.logger.info(f"[{count_add} of {len(add_list_keys)}] users added.")
            
            
        
        return count_add

class Workday:
    def build_comparison_string(self, wd_row, locs, loc_map_table):
        loc_id = cal_user_location(wd_row, locs, loc_map_table)
        hire_date = wd_row['hire_date'].split('-')
        
        is_manager = "supervisor" if wd_row.get("is_manager", "") else "non_supervisor"
        return (
            wd_row["primary_work_email"]
            + "|"
            + wd_row["preferred_first_name"]
            + "|"
            + wd_row["preferred_last_name"]
            + "|"
            + wd_row["employee_id"]
            + "|"
            + loc_id
            + "|"
            + is_manager
            + "|"
            + hire_date[1] + "-" + hire_date[0]
            + "|"
            + wd_row["primary_work_email"]
        )

    def get_wd_users(self, locs, loc_map_table):
        import pandas as pd
        import io

        # The API is not returning all fields in the json
        # but the csv is, so we will use the csv version
        # wd_users_csv = WorkdayAPI.get_datawarehouse_workers_csv()
        workday_api = WorkdayAPI()
        wd_users_csv = workday_api.get_datawarehouse_workers_csv()
        df = pd.read_csv(io.StringIO(wd_users_csv), sep=",")
        filtered = df[
            (df["currently_active"] == True)
            & (df["moco_or_mofo"] == "MoCo")
            & (df["worker_type"] == "Employee")
            | (df['primary_work_email'] == "jmoscon@mozilla.com")
        ]

        comp = {
            x[1]["primary_work_email"]: self.build_comparison_string(
                x[1], locs, loc_map_table
            )
            for x in filtered.iterrows()
        }
        return comp, {x[1]["primary_work_email"]: x for x in filtered.iterrows()}


class WorkdayEverfiIntegration:
    def __init__(self) -> None:
        self.workday = Workday()
        self.everfi = Everfi()
        self.logger = logging.getLogger(self.__class__.__name__)

    def compare_users(self, wd_comp, everfi_comp, wd_users, everfi_users):
        import numpy as np

        add_list = []
        del_list = []
        upd_list = []
        wd_users_emails = list(wd_users.keys())
        everfi_users_emails = list(everfi_users.keys())
        add_list = np.setdiff1d(wd_users_emails, everfi_users_emails)
        del_list = np.setdiff1d(everfi_users_emails, wd_users_emails)
        intersect_list = np.intersect1d(wd_users_emails, everfi_users_emails)

        for upd_email in intersect_list:
            if everfi_comp[upd_email] != wd_comp[upd_email]:
                upd_list.append(upd_email)

 
        return add_list, del_list, upd_list

    def run(self, limit):
        # ========================================================
        # Getting Everfi hire dates, locations and locations mapping table ...
        # ========================================================        
        try:
            self.logger.info("Getting everfi hire dates")
            hire_date_category_id, hire_dates = self.everfi.everfi_api.get_hire_dates()           
            self.logger.info(f"Number of hire dates: {len(hire_dates)}")

            self.logger.info("Getting everfi locations")
            locs = self.everfi.everfi_api.get_locations()
            self.logger.info(f"Number of locations: {len(locs)}")

            self.logger.info("Getting everfi mapping table")
            loc_map_table = self.everfi.everfi_api.get_locations_mapping_table()
            self.logger.info(f"Number of mappins: {len(loc_map_table)}")

        except (APIAdaptorException, Exception) as e:
            self.logger.error(str(e))
            self.logger.critical("Failed while Getting Everfi hire dates,locations and locations mapping table ...")            
            sys.exit(1)

        # ========================================================
        # Getting Workday users...
        # ========================================================
        self.logger.info("Getting Workday users...")
        try:
            wd_comp, wd_users = self.workday.get_wd_users(locs, loc_map_table)
            self.logger.info(f"Number of wd users: {len(wd_users)}")
        except (APIAdaptorException, Exception) as e:
            self.logger.error(str(e))
            self.logger.critical("Failed while Getting Workday users...")           
            sys.exit(1)

        # ========================================================
        # Getting Everfi users...
        # ========================================================
        self.logger.info("Getting Everfi users...")
        try:
            everfi_comp, everfi_users = self.everfi.get_everfi_users(locs, loc_map_table, hire_dates)
            self.logger.info(f"Number of Everfi users: {len(everfi_users)}")
        except (APIAdaptorException, Exception) as e:
            self.logger.error(str(e))
            self.logger.critical("Failed while Getting Everfi users...")
            sys.exit(1)
            
        # ========================================================
        # Comparing users...
        # ========================================================
        self.logger.info("Comparing users...")
        try:
            add_list, del_list, upd_list = integration.compare_users(
                wd_comp, everfi_comp, wd_users, everfi_users
            )

            self.logger.info(f"Number of users to delete w/o limit={len(del_list)} with limit={len(del_list[:limit])}")
            self.logger.info(f"Number of users to add w/o limit={len(add_list)} with limit={len(add_list[:limit])}")
            self.logger.info(f"Number of users to update w/o limit={len(upd_list)} with limit={len(upd_list[:limit])}")

            del_list = del_list[:limit]
            add_list = add_list[:limit]
            upd_list = upd_list[:limit]
  
        except (Exception) as e:
            self.logger.error(str(e))
            self.logger.critical("Failed while Comparing users...")
            sys.exit(1)
        
            
        # ========================================================
        # Deleting Everfi users ...
        # ========================================================
        self.logger.info("Deleting Everfi users ...")        
        try:
             
            count_dels = self.everfi.deactivate_users(del_list, everfi_users)
            self.logger.info(f"Number of users deleted {count_dels}")
        except (APIAdaptorException, Exception) as e:
            self.logger.error(str(e))
            self.logger.critical("Faile while Deleting Everfi users ...")
            sys.exit(1)
            
        # ========================================================
        # Adding Everfi users ...
        # ========================================================
        self.logger.info("Adding Everfi users ...")
        try:
            count_add = self.everfi.add_everfi_users(
                hire_date_category_id, hire_dates, locs, add_list, wd_users, loc_map_table
            )
            self.logger.info(f"Number of users added {count_add}")            
        except (APIAdaptorException, Exception) as e:
            self.logger.error(str(e))
            self.logger.critical("Failed while Adding Everfi users ...")
            sys.exit(1)
        # ========================================================
        # Updating Everfi users ...
        # ========================================================
        self.logger.info("Updating Everfi users ...")
        
        try:
            count_upd = self.everfi.upd_everfi_users(
                hire_date_category_id,
                hire_dates,
                locs,
                upd_list,
                wd_users,
                everfi_users,
                loc_map_table,
            )
            self.logger.info(f"Number of users updated {count_upd}")
        except (APIAdaptorException, Exception) as e:
            self.logger.error(str(e))
            self.logger.critical("Failed while Updating Everfi users ...")
            sys.exit(1)
        
        self.logger.info("End of integration")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Sync up XMatters with Workday")

    parser.add_argument(
        "-l",
        "--level",
        action="store",
        help="log level (debug, info, warning, error, or critical)",
        type=str,
        default="info",
    )
   
    parser.add_argument(
        "-m",
        "--max_limit", 
        action="store",
        type=int,
        help="limit the number of changes in Everfi",        
        default=40
    )
    args = None
    args = parser.parse_args()
    
    log_level = Util.set_up_logging(args.level)

    logger = logging.getLogger(__name__)

    logger.info("Starting...")
    logger.info(f"max_limit={args.max_limit}")

    integration = WorkdayEverfiIntegration()

    integration.run(args.max_limit)