in components/places/src/bookmark_sync/engine.rs [466:669]
fn apply_remote_items(db: &PlacesDb, scope: &SqlInterruptScope, now: Timestamp) -> Result<()> {
// Remove all keywords from old and new URLs, and remove new keywords
// from all existing URLs. The `NOT NULL` conditions are important; they
// ensure that SQLite uses our partial indexes on `itemsToApply`,
// instead of a table scan.
log::debug!("Removing old keywords");
scope.err_if_interrupted()?;
db.execute_batch(
"DELETE FROM moz_keywords
WHERE place_id IN (SELECT oldPlaceId FROM itemsToApply
WHERE oldPlaceId NOT NULL) OR
place_id IN (SELECT newPlaceId FROM itemsToApply
WHERE newPlaceId NOT NULL) OR
keyword IN (SELECT newKeyword FROM itemsToApply
WHERE newKeyword NOT NULL)",
)?;
log::debug!("Removing old tags");
scope.err_if_interrupted()?;
db.execute_batch(
"DELETE FROM moz_tags_relation
WHERE place_id IN (SELECT oldPlaceId FROM itemsToApply
WHERE oldPlaceId NOT NULL) OR
place_id IN (SELECT newPlaceId FROM itemsToApply
WHERE newPlaceId NOT NULL)",
)?;
// Due to bug 1935797, we try to add additional logging on what exact
// guids are colliding as it could shed light on what's going on
log::debug!("Checking for potential GUID collisions before upserting items");
let collision_check_sql = "
SELECT ia.localId, ia.mergedGuid, ia.remoteGuid, b.id, b.guid
FROM itemsToApply ia
JOIN moz_bookmarks b ON ia.mergedGuid = b.guid
WHERE (ia.localId IS NULL OR ia.localId != b.id)
";
let potential_collisions: Vec<(Option<i64>, String, String, i64, String)> = db
.prepare(collision_check_sql)?
.query_map([], |row| {
let ia_local_id: Option<i64> = row.get(0)?;
let ia_merged_guid: String = row.get(1)?;
let ia_remote_guid: String = row.get(2)?;
let bmk_id: i64 = row.get(3)?;
let bmk_guid: String = row.get(4)?;
Ok((
ia_local_id,
ia_merged_guid,
ia_remote_guid,
bmk_id,
bmk_guid,
))
})?
.filter_map(|entry| entry.ok())
.collect();
if !potential_collisions.is_empty() {
// Log details about the collisions
for (ia_local_id, ia_merged_guid, ia_remote_guid, bmk_id, bmk_guid) in &potential_collisions
{
error_support::breadcrumb!(
"Found GUID collision: ia_localId={:?}, ia_mergedGuid={}, ia_remoteGuid={}, mb_id={}, mb_guid={}",
ia_local_id,
ia_merged_guid,
ia_remote_guid,
bmk_id,
bmk_guid
);
}
}
// Due to bug 1935797, we need to check if any users have any
// undetected orphaned bookmarks and report them
let orphaned_count: i64 = db.query_row(
"WITH RECURSIVE orphans(id) AS (
SELECT b.id
FROM moz_bookmarks b
WHERE b.parent IS NOT NULL
AND NOT EXISTS (
SELECT 1 FROM moz_bookmarks p WHERE p.id = b.parent
)
UNION
SELECT c.id
FROM moz_bookmarks c
JOIN orphans o ON c.parent = o.id
)
SELECT COUNT(*) FROM orphans;",
[],
|row| row.get(0),
)?;
if orphaned_count > 0 {
log::warn!("Found {} orphaned bookmarks during sync", orphaned_count);
error_support::breadcrumb!(
"places-sync-bookmarks-orphaned: found local orphans before upsert {}",
orphaned_count
);
}
// Insert and update items, temporarily using the Places root for new
// items' parent IDs, and -1 for positions. We'll fix these up later,
// when we apply the new local structure. This `INSERT` is a full table
// scan on `itemsToApply`. The no-op `WHERE` clause is necessary to
// avoid a parsing ambiguity.
log::debug!("Upserting new items");
let upsert_sql = format!(
"INSERT INTO moz_bookmarks(id, guid, parent,
position, type, fk, title,
dateAdded,
lastModified,
syncStatus, syncChangeCounter)
SELECT localId, mergedGuid, (SELECT id FROM moz_bookmarks
WHERE guid = '{root_guid}'),
-1, {type_fragment}, newPlaceId, newTitle,
/* Pick the older of the local and remote date added. We'll
weakly reupload any items with an older local date. */
MIN(IFNULL(localDateAdded, remoteDateAdded), remoteDateAdded),
/* The last modified date should always be newer than the date
added, so we pick the newer of the two here. */
MAX(lastModified, remoteDateAdded),
{sync_status}, 0
FROM itemsToApply
WHERE 1
ON CONFLICT(id) DO UPDATE SET
title = excluded.title,
dateAdded = excluded.dateAdded,
lastModified = excluded.lastModified,
fk = excluded.fk,
syncStatus = {sync_status}
/* Due to bug 1935797, we found scenarios where users had bookmarks with GUIDs that matched
* incoming records BUT for one reason or another dogear doesn't believe it exists locally
* This handles the case where we try to insert a new bookmark with a GUID that already exists,
* updating the existing record instead of failing with a constraint violation.
* Usually the above conflict will catch most of these scenarios and there's no issue of
* any dupes being added here since users that hit this before would've just failed the bookmark sync
*/
ON CONFLICT(guid) DO UPDATE SET
title = excluded.title,
dateAdded = excluded.dateAdded,
lastModified = excluded.lastModified,
fk = excluded.fk,
syncStatus = {sync_status}",
root_guid = BookmarkRootGuid::Root.as_guid().as_str(),
type_fragment = ItemTypeFragment("newKind"),
sync_status = SyncStatus::Normal as u8,
);
scope.err_if_interrupted()?;
let result = db.execute_batch(&upsert_sql);
// In trying to debug bug 1935797 - relaxing the trigger caused a spike on
// guid collisions, we want to report on this during the upsert to see
// if we can discern any obvious signs
if let Err(rusqlite::Error::SqliteFailure(e, _)) = &result {
if e.code == ErrorCode::ConstraintViolation {
error_support::report_error!(
"places-sync-bookmarks-constraint-violation",
"Hit a constraint violation {:?}",
result
);
}
}
// Return the original result
result?;
log::debug!("Flagging frecencies for recalculation");
scope.err_if_interrupted()?;
db.execute_batch(&format!(
"REPLACE INTO moz_places_stale_frecencies(place_id, stale_at)
SELECT oldPlaceId, {now} FROM itemsToApply
WHERE newKind = {bookmark_kind} AND (
oldPlaceId IS NULL <> newPlaceId IS NULL OR
oldPlaceId <> newPlaceId
)
UNION ALL
SELECT newPlaceId, {now} FROM itemsToApply
WHERE newKind = {bookmark_kind} AND (
newPlaceId IS NULL <> oldPlaceId IS NULL OR
newPlaceId <> oldPlaceId
)",
now = now,
bookmark_kind = SyncedBookmarkKind::Bookmark as u8,
))?;
log::debug!("Inserting new keywords for new URLs");
scope.err_if_interrupted()?;
db.execute_batch(
"INSERT OR IGNORE INTO moz_keywords(keyword, place_id)
SELECT newKeyword, newPlaceId
FROM itemsToApply
WHERE newKeyword NOT NULL",
)?;
log::debug!("Inserting new tags for new URLs");
scope.err_if_interrupted()?;
db.execute_batch(
"INSERT OR IGNORE INTO moz_tags_relation(tag_id, place_id)
SELECT r.tagId, n.newPlaceId
FROM itemsToApply n
JOIN moz_bookmarks_synced_tag_relation r ON r.itemId = n.remoteId",
)?;
Ok(())
}