in ees_sharepoint/sync_sharepoint.py [0:0]
def fetch_items(self, lists, ids):
"""This method fetches items from all the lists in a collection and
invokes theindex permission method to get the document level permissions.
If the fetching is not successful, it logs proper message.
:param lists: document lists
:param ids: structure containing id's of all objects
Returns:
document: response of sharepoint GET call, with fields specified in the schema
"""
responses = []
# here value is a list of url and title
self.logger.info("Fetching all the items for the lists")
if not lists:
self.logger.info(
"No item was created in this interval: start time: %s and end time: %s"
% (self.start_time, self.end_time)
)
else:
for value in lists.values():
if not ids["list_items"].get(value[0]):
ids["list_items"].update({value[0]: {}})
schema_item = self.get_schema_fields(LIST_ITEMS)
for list_content, value in lists.items():
if parse(self.start_time) > parse(value[2]):
continue
rel_url = f"{value[0]}/_api/web/lists(guid'{list_content}')/items?$select=*,FileRef"
self.logger.info(
"Fetching the items for list: %s from url: %s" % (value[1], rel_url)
)
query = self.sharepoint_client.get_query(
self.start_time, self.end_time, LIST_ITEMS
)
response = self.sharepoint_client.get(rel_url, query, LIST_ITEMS)
response_data = get_results(self.logger, response, LIST_ITEMS)
if not response_data:
self.logger.info(
"No item was created for the list %s in this interval: start time: %s and end time: %s"
% (value[1], self.start_time, self.end_time)
)
continue
self.logger.info(
"Successfully fetched and parsed %s listitem response for list: %s from SharePoint"
% (len(response_data), value[1])
)
document = []
if not ids["list_items"][value[0]].get(list_content):
ids["list_items"][value[0]].update({list_content: []})
rel_url = f"{value[0]}/_api/web/lists(guid'{list_content}')/items?$select=Attachments,AttachmentFiles,Title&$expand=AttachmentFiles"
file_response_data = self.sharepoint_client.get(
rel_url, query=query, param_name="attachment"
)
if file_response_data:
file_response_data = get_results(
self.logger, file_response_data.json(), "attachment"
)
for i, _ in enumerate(response_data):
doc = {"type": ITEM}
if response_data[i].get("Attachments") and file_response_data:
for data in file_response_data:
if response_data[i].get("Title") == data["Title"]:
file_relative_url = data["AttachmentFiles"]["results"][
0
]["ServerRelativeUrl"]
url_s = f"{value[0]}/_api/web/GetFileByServerRelativeUrl('{encode(file_relative_url)}')/$value"
response = self.sharepoint_client.get(
url_s, query="", param_name="attachment"
)
doc["body"] = {}
if response and response.ok:
try:
doc["body"] = extract(response.content)
except TikaException as exception:
self.logger.error(
"Error while extracting the contents from the attachment, Error %s"
% (exception)
)
break
for field, response_field in schema_item.items():
doc[field] = response_data[i].get(response_field)
if self.enable_permission is True:
doc["_allow_permissions"] = self.fetch_permissions(
key=LIST_ITEMS,
list_id=list_content,
list_url=value[0],
itemid=str(response_data[i]["Id"]),
)
relative_url = response_data[i].get("FileRef")
doc["url"] = urljoin(self.sharepoint_host, relative_url)
document.append(doc)
if (
response_data[i].get("GUID")
not in ids["list_items"][value[0]][list_content]
):
ids["list_items"][value[0]][list_content].append(
response_data[i].get("GUID")
)
responses.extend(document)
documents = {"type": LIST_ITEMS, "data": responses}
return documents