in ees_sharepoint/checkpointing.py [0:0]
def get_checkpoint(self, collection, current_time):
"""This method fetches the checkpoint from the checkpoint file in
the local storage. If the file does not exist, it takes the
checkpoint details from the configuration file.
:param collection: collection name
:param current_time: current time"""
self.logger.info(
"Fetching the checkpoint details from the checkpoint file: %s"
% CHECKPOINT_PATH
)
if os.path.exists(CHECKPOINT_PATH) and os.path.getsize(CHECKPOINT_PATH) > 0:
self.logger.debug(
"Checkpoint file exists and has contents, hence considering the checkpoint time instead of start_time and end_time"
)
with open(CHECKPOINT_PATH) as checkpoint_store:
try:
checkpoint_list = json.load(checkpoint_store)
if not checkpoint_list.get(collection):
self.logger.info(
f"""Checkpoint file is present but does not contain start_time \
for the collection #{collection}. Using start_time and end_time \
from the configuration file instead of the last successful fetch time"""
)
start_time = self.config.get_value("start_time")
end_time = self.config.get_value("end_time")
else:
self.logger.info(
"Considering the start_time from the checkpoint"
)
start_time = checkpoint_list.get(collection)
end_time = current_time
except ValueError as exception:
self.logger.exception(
"Error while parsing the json file of the checkpoint store from path: %s. Error: %s"
% (CHECKPOINT_PATH, exception)
)
self.logger.info(
"Considering the start_time and end_time from the configuration file"
)
start_time = self.config.get_value("start_time")
end_time = self.config.get_value("end_time")
else:
self.logger.debug(
"Checkpoint file does not exist at %s, considering the start_time and end_time from the configuration file"
% CHECKPOINT_PATH
)
start_time = self.config.get_value("start_time")
end_time = self.config.get_value("end_time")
self.logger.debug(
"Contents of the start_time: %s and end_time: %s for collection %s",
start_time,
end_time,
collection,
)
return start_time, end_time