ees_sharepoint/deletion_sync_command.py

#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License 2.0;
# you may not use this file except in compliance with the Elastic License 2.0.
#
"""This module allows to remove recently deleted documents from Elastic Enterprise Search.

Documents that were deleted in Sharepoint Server instance will still be available in
Elastic Enterprise Search until a full sync happens, or until this module is used."""

import json
import os

import requests

from .base_command import BaseCommand
from .utils import split_list_into_buckets

IDS_PATH = os.path.join(os.path.dirname(__file__), 'doc_id.json')
# By default, Enterprise Search configuration has a maximum allowed limit set to 100 documents for an api request
BATCH_SIZE = 100


class DeletionSyncCommand(BaseCommand):
    """DeletionSyncCommand class allows to remove instances of specific Sharepoint Object types.

    It provides a way to remove items, lists and sites from Elastic Enterprise Search
    that were deleted in Sharepoint Server instance."""

    def __init__(self, args):
        super().__init__(args)

    def deindexing_items(self, collection, ids, key):
        """Fetches the id's of deleted items from the sharepoint server and
           invokes delete documents api for those ids to remove them from
           workplace search"""
        logger = self.logger
        delete_ids_items = ids["delete_keys"][collection].get(key)

        logger.info(f"Deindexing {key}...")
        if delete_ids_items:
            delete_site = []
            global_ids_items = ids["global_keys"][collection][key]
            for site_url, item_details in delete_ids_items.items():
                delete_list = []
                for list_id, items in item_details.items():
                    doc = []
                    for item_id in items:
                        url = f"{site_url}/_api/web/lists(guid\'{list_id}\')/items"
                        resp = self.sharepoint_client.get(
                            url, f"?$filter= GUID eq  \'{item_id}\'", "deindex")
                        if resp:
                            response = resp.json()
                            result = response.get('d', {}).get('results')
                        if resp.status_code == requests.codes['not_found'] or result == []:
                            doc.append(item_id)
                    if doc:
                        for chunk in split_list_into_buckets(doc, BATCH_SIZE):
                            self.workplace_search_custom_client.delete_documents(
                                document_ids=chunk)
                    updated_items = global_ids_items[site_url].get(list_id)
                    if updated_items is None:
                        continue
                    for updated_item_id in doc:
                        if updated_item_id in updated_items:
                            updated_items.remove(updated_item_id)
                    if updated_items == []:
                        delete_list.append(list_id)
                for list_id in delete_list:
                    global_ids_items[site_url].pop(list_id)
                if global_ids_items[site_url] == {}:
                    delete_site.append(site_url)
            for site_url in delete_site:
                global_ids_items.pop(site_url)
        else:
            logger.info("No %s found to be deleted for collection: %s" % (key, collection))
        return ids

    def deindexing_lists(self, collection, ids):
        """Fetches the id's of deleted lists from the sharepoint server and
            further removes them from workplace search by invoking the delete
            document api.
            Returns:
                sites: list of site paths
        """
        logger = self.logger

        delete_ids_lists = ids["delete_keys"][collection].get('lists')
        logger.info("Deindexing lists...")
        if delete_ids_lists:
            delete = []
            global_ids_lists = ids["global_keys"][collection]["lists"]
            for site_url, list_details in delete_ids_lists.items():
                doc = []
                for list_id in list_details.keys():
                    url = f"{site_url}/_api/web/lists(guid\'{list_id}\')"
                    resp = self.sharepoint_client.get(url, '', "deindex")
                    if resp is not None and resp.status_code == requests.codes['not_found']:
                        doc.append(list_id)
                for chunk in split_list_into_buckets(doc, BATCH_SIZE):
                    self.workplace_search_custom_client.delete_documents(
                        document_ids=chunk)
                for list_id in doc:
                    if list_id in global_ids_lists[site_url]:
                        global_ids_lists[site_url].pop(list_id)
                if global_ids_lists[site_url] == {}:
                    delete.append(site_url)
            for site_url in delete:
                global_ids_lists.pop(site_url)
        else:
            logger.info("No list found to be deleted for collection: %s" % collection)
        return ids

    def deindexing_sites(self, collection, ids):
        """Fetches the ids' of deleted sites from the sharepoint server and
            invokes delete documents api for those ids to remove them from
            workplace search
        """
        logger = self.logger

        site_details = ids["delete_keys"][collection].get("sites")
        logger.info("Deindexing sites...")
        if site_details:
            doc = []
            for site_id, site_url in site_details.items():
                url = f"{site_url}/_api/web"
                resp = self.sharepoint_client.get(url, '', "deindex")
                if resp is not None and resp.status_code == requests.codes['not_found']:
                    doc.append(site_id)
            for chunk in split_list_into_buckets(doc, BATCH_SIZE):
                self.workplace_search_custom_client.delete_documents(
                    document_ids=chunk)
            for site_id in doc:
                ids["global_keys"][collection]["sites"].pop(site_id)
        else:
            logger.info("No sites found to be deleted for collection: %s" % collection)
        return ids

    def execute(self):
        """Runs the deletion sync logic"""
        logger = self.logger
        logger.info("Running deletion sync")

        try:
            with open(IDS_PATH) as file:
                ids = json.load(file)
            for collection in self.config.get_value('sharepoint.site_collections'):
                logger.info(
                    'Starting the deindexing for site collection: %s' % collection)
                if ids["delete_keys"].get(collection):
                    ids = self.deindexing_sites(collection, ids)
                    ids = self.deindexing_lists(collection, ids)
                    ids = self.deindexing_items(collection, ids, "list_items")
                    ids = self.deindexing_items(collection, ids, "drive_items")
                else:
                    logger.info("No objects present to be deleted for the collection: %s" % collection)
            ids["delete_keys"] = {}
            with open(IDS_PATH, "w") as file:
                try:
                    json.dump(ids, file, indent=4)
                except ValueError as exception:
                    logger.exception(
                        "Error while updating the doc_id json file. Error: %s", exception
                    )
        except FileNotFoundError as exception:
            logger.warning(
                "[Fail] File doc_id.json is not present, none of the objects are indexed. Error: %s"
                % exception
            )