in bedrock/contentful/management/commands/update_contentful.py [0:0]
def _refresh_from_contentful(self) -> tuple[int, int, int, int]:
self.log("Pulling from Contentful")
updated_count = 0
added_count = 0
deleted_count = 0
error_count = 0
content_missing_localised_version = set()
EMPTY_ENTRY_ATTRIBUTE_STRING = "'Entry' object has no attribute 'content'"
available_locales = ContentfulPage.client.locales()
# 1. Build a lookup of pages to sync by type, ID and locale
content_to_sync = self._get_content_to_sync(available_locales)
# 2. Pull down each page and store
# TODO: we may (TBC) be able to do a wholesale refactor and get all the locale variations
# of a single Page (where entry['myfield'] in a single-locale setup changes to
# entry['myfield']['en-US'], entry['myfield']['de'], etc. That might be particularly useful
# when we have a lot of locales in play. For now, the heavier-IO approach should be OK.
for ctype, page_id, locale_code in content_to_sync:
request = self.rf.get("/")
request.locale = locale_code
try:
page = ContentfulPage(request, page_id)
page_data = page.get_content()
except AttributeError as ae:
# Problem with the page - most likely not-really-a-page-in-this-locale-after-all.
# (Contentful seems to send back a Compose `page` in en-US for _any_ other locale,
# even if the page has no child entries. This false positive / absent entry is
# only apparent when we try to call page.get_content() and find there is none.)
if str(ae) == EMPTY_ENTRY_ATTRIBUTE_STRING:
self.log(f"No content for {page_id} for {locale_code} - page will be deleted from DB if it exists")
# We want to track this explicitly, because we need to do cleanup later on.
content_missing_localised_version.add((ctype, page_id, locale_code))
continue
else:
raise
except Exception as ex:
# Problem with the page, load other pages
self.log(f"Problem with {ctype}:{page_id} -> {type(ex)}: {ex}")
capture_exception(ex)
error_count += 1
continue
hash = data_hash(page_data)
_info = page_data["info"]
# Check we're definitely getting the locales we're expecting (with a temporary caveat)
if (
locale_code != _info["locale"]
and
# Temporary workaround till Homepage moves into Compose from Connect
page_id not in settings.CONTENTFUL_HOMEPAGE_LOOKUP.values()
):
msg = f"Locale mismatch on {ctype}:{page_id} -> {locale_code} vs {_info['locale']}"
self.log(msg)
capture_message(msg)
error_count += 1
continue
# Now we've done the check, let's convert any Contentful-specific
# locale name into one we use in Bedrock before it reaches the database
_info["locale"] = self._remap_locale_for_bedrock(_info["locale"])
extra_params = dict(
locale=_info["locale"],
data_hash=hash,
data=page_data,
slug=_info["slug"],
classification=_info.get("classification", ""),
tags=_info.get("tags", []),
category=_info.get("category", ""),
)
try:
obj = ContentfulEntry.objects.get(
contentful_id=page_id,
locale=_info["locale"],
)
except ContentfulEntry.DoesNotExist:
self.log(f"Creating new ContentfulEntry for {ctype}:{locale_code}:{page_id}")
ContentfulEntry.objects.create(
contentful_id=page_id,
content_type=ctype,
**extra_params,
)
added_count += 1
else:
if self.force or hash != obj.data_hash:
self.log(f"Updating existing ContentfulEntry for {ctype}:{locale_code}:{page_id}")
for key, value in extra_params.items():
setattr(obj, key, value)
obj.last_modified = tz_now()
obj.save()
updated_count += 1
try:
# Even if we failed to sync certain entities that are usually syncable, we
# should act as if they were synced when we come to look for records to delete.
# (If it was just a temporary glitch that caused the exception we would not
# want to unncessarily delete a page, even if the failed sync means its content
# is potentially stale)
# HOWEVER, there are some entities which are just not syncable at all - such as
# a Compose `page` which has no entry for a specific locale, and so is skipped
# above. For these, we DO want to delete them, so remove them from the list of
# synced items
entries_processed_in_sync = set(content_to_sync).difference(content_missing_localised_version)
deleted_count = self._detect_and_delete_absent_entries(entries_processed_in_sync)
except Exception as ex:
self.log(ex)
capture_exception(ex)
self._check_localisation_complete()
return added_count, updated_count, deleted_count, error_count