in ees_network_drive/sync_network_drives.py [0:0]
def perform_sync(self, drive, partition_paths):
"""This method fetches all the objects from Network Drives server and
appends them to the shared queue
:param drive: The Network Drive name
:param indexing_rules: Object of the indexing rules
Returns:
storage: dictionary containing the ids and path of all the files in Network Drives
"""
if not partition_paths:
return {}
files = Files(self.logger, self.config, self.network_drive_client)
documents_to_index = []
self.logger.info(f"Thread: [{threading.get_ident()}] fetching all the files for folder {partition_paths}")
ids_storage = {}
try:
fetched_documents = files.fetch_files(
self.drive_path.parts[0],
partition_paths,
self.time_range,
self.indexing_rules,
)
self.queue.append_to_queue(fetched_documents)
documents_to_index.extend(fetched_documents)
except Exception as exception:
self.logger.error(f"Error while fetching files for the path: {partition_paths}. Error: {exception}")
for doc in documents_to_index:
ids_storage.update({doc["id"]: doc["path"]})
return ids_storage