sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql (615 lines of code) (raw):
WITH events_unnested AS (
SELECT
DATE(submission_timestamp) AS submission_date,
category AS event_category,
name AS event_name,
timestamp AS event_timestamp,
client_info,
METADATA,
normalized_os,
normalized_os_version,
normalized_country_code,
normalized_channel,
ping_info,
extra AS event_details,
metrics
FROM
-- https://dictionary.telemetry.mozilla.org/apps/firefox_desktop/pings/newtab
`moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1`,
UNNEST(events)
WHERE
DATE(submission_timestamp) = @submission_date
AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket')
),
visit_metadata AS (
SELECT
mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
submission_date,
ANY_VALUE(client_info.client_id) AS client_id,
ANY_VALUE(metrics.uuid.legacy_telemetry_client_id) AS legacy_telemetry_client_id,
ANY_VALUE(metrics.uuid.legacy_telemetry_profile_group_id) AS profile_group_id,
ANY_VALUE(normalized_os) AS normalized_os,
ANY_VALUE(normalized_os_version) AS normalized_os_version,
ANY_VALUE(normalized_country_code) AS country_code,
ANY_VALUE(normalized_channel) AS channel,
ANY_VALUE(client_info.locale) AS locale,
ANY_VALUE(client_info.app_display_version) AS browser_version,
"Firefox Desktop" AS browser_name,
ANY_VALUE(metrics.string.search_engine_default_engine_id) AS default_search_engine,
ANY_VALUE(metrics.string.search_engine_private_engine_id) AS default_private_search_engine,
ANY_VALUE(metrics.boolean.pocket_is_signed_in) AS pocket_is_signed_in,
ANY_VALUE(metrics.boolean.pocket_enabled) AS pocket_enabled,
ANY_VALUE(metrics.boolean.pocket_sponsored_stories_enabled) AS pocket_sponsored_stories_enabled,
ANY_VALUE(metrics.boolean.topsites_enabled) AS topsites_enabled,
ANY_VALUE(metrics.boolean.topsites_sponsored_enabled) AS topsites_sponsored_enabled,
ANY_VALUE(
metrics.quantity.topsites_sponsored_tiles_configured
) AS topsites_sponsored_tiles_configured,
ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category,
ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category,
ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled,
ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled,
ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows,
ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors,
ANY_VALUE(ping_info.experiments) AS experiments,
MIN(IF(event_name = "opened", event_timestamp, NULL)) AS newtab_visit_started_at,
MIN(IF(event_name = "closed", event_timestamp, NULL)) AS newtab_visit_ended_at,
ANY_VALUE(
IF(event_name = "opened", mozfun.map.get_key(event_details, "source"), NULL)
) AS newtab_open_source,
LOGICAL_OR(event_name IN ("click", "issued", "save")) AS had_non_impression_engagement,
LOGICAL_OR(event_name IN ("click", "save")) AS had_non_search_engagement,
ANY_VALUE(metrics.string_list.newtab_selected_topics) AS newtab_selected_topics,
ANY_VALUE(
IF(
event_name = "opened",
SAFE_CAST(mozfun.map.get_key(event_details, "window_inner_height") AS INT),
NULL
)
) AS newtab_window_inner_height,
ANY_VALUE(
IF(
event_name = "opened",
SAFE_CAST(mozfun.map.get_key(event_details, "window_inner_width") AS INT),
NULL
)
) AS newtab_window_inner_width,
FROM
events_unnested
GROUP BY
newtab_visit_id,
submission_date
HAVING
newtab_visit_id IS NOT NULL
),
search_events AS (
SELECT
mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
`moz-fx-data-shared-prod`.udf.normalize_search_engine(
mozfun.map.get_key(event_details, "telemetry_id")
) AS search_engine,
mozfun.map.get_key(event_details, "search_access_point") AS search_access_point,
COUNTIF(event_name = "issued") AS searches,
COUNTIF(
event_name = "impression"
AND mozfun.map.get_key(event_details, "is_tagged") = "true"
) AS tagged_search_ad_impressions,
COUNTIF(
event_name = "impression"
AND mozfun.map.get_key(event_details, "is_follow_on") = "true"
) AS follow_on_search_ad_impressions,
COUNTIF(
event_name = "impression"
AND mozfun.map.get_key(event_details, "is_follow_on") = "true"
AND mozfun.map.get_key(event_details, "is_tagged") = "true"
) AS tagged_follow_on_search_ad_impressions,
COUNTIF(
event_name = "click"
AND mozfun.map.get_key(event_details, "is_tagged") = "true"
) AS tagged_search_ad_clicks,
COUNTIF(
event_name = "click"
AND mozfun.map.get_key(event_details, "is_follow_on") = "true"
) AS follow_on_search_ad_clicks,
COUNTIF(
event_name = "click"
AND mozfun.map.get_key(event_details, "is_follow_on") = "true"
AND mozfun.map.get_key(event_details, "is_tagged") = "true"
) AS tagged_follow_on_search_ad_clicks
FROM
events_unnested
WHERE
event_category IN ('newtab.search', 'newtab.search.ad')
AND event_name IN ('click', 'impression', 'issued')
GROUP BY
newtab_visit_id,
search_engine,
search_access_point
),
search_summary AS (
SELECT
newtab_visit_id,
ARRAY_AGG(
STRUCT(
search_engine,
search_access_point,
searches,
tagged_search_ad_clicks,
tagged_search_ad_impressions,
follow_on_search_ad_clicks,
follow_on_search_ad_impressions,
tagged_follow_on_search_ad_clicks,
tagged_follow_on_search_ad_impressions
)
) AS search_interactions
FROM
search_events
GROUP BY
newtab_visit_id
),
topsites_events AS (
SELECT
mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS topsite_tile_position,
mozfun.map.get_key(event_details, "advertiser_name") AS topsite_tile_advertiser_name,
mozfun.map.get_key(event_details, "tile_id") AS topsite_tile_id,
JSON_EXTRACT(sov, "$.assigned") AS topsite_tile_assigned_sov_branch,
JSON_EXTRACT(sov, "$.chosen") AS topsite_tile_displayed_sov_branch,
COUNTIF(event_name = 'impression') AS topsite_tile_impressions,
COUNTIF(event_name = 'click') AS topsite_tile_clicks,
COUNTIF(
event_name = 'impression'
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
) AS sponsored_topsite_tile_impressions,
COUNTIF(
event_name = 'impression'
AND mozfun.map.get_key(event_details, "is_sponsored") = "false"
) AS organic_topsite_tile_impressions,
COUNTIF(
event_name = 'click'
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
) AS sponsored_topsite_tile_clicks,
COUNTIF(
event_name = 'click'
AND mozfun.map.get_key(event_details, "is_sponsored") = "false"
) AS organic_topsite_tile_clicks,
COUNTIF(event_name = 'dismiss') AS topsite_tile_dismissals,
COUNTIF(
event_name = 'dismiss'
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
) AS sponsored_topsite_tile_dismissals,
COUNTIF(
event_name = 'dismiss'
AND mozfun.map.get_key(event_details, "is_sponsored") = "false"
) AS organic_topsite_tile_dismissals,
FROM
events_unnested
LEFT JOIN
UNNEST(metrics.string_list.newtab_sov_allocation) sov
ON SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) = SAFE_CAST(
JSON_EXTRACT(sov, "$.pos") AS INT64
)
WHERE
event_category = 'topsites'
AND event_name IN ('dismiss', 'click', 'impression')
GROUP BY
newtab_visit_id,
topsite_tile_position,
topsite_tile_advertiser_name,
topsite_tile_id,
topsite_tile_assigned_sov_branch,
topsite_tile_displayed_sov_branch
),
topsites_summary AS (
SELECT
newtab_visit_id,
ARRAY_AGG(
STRUCT(
topsite_tile_advertiser_name,
topsite_tile_position,
topsite_tile_id,
topsite_tile_assigned_sov_branch,
topsite_tile_displayed_sov_branch,
topsite_tile_clicks,
sponsored_topsite_tile_clicks,
organic_topsite_tile_clicks,
topsite_tile_impressions,
sponsored_topsite_tile_impressions,
organic_topsite_tile_impressions,
topsite_tile_dismissals,
sponsored_topsite_tile_dismissals,
organic_topsite_tile_dismissals
)
) AS topsite_tile_interactions
FROM
topsites_events
GROUP BY
newtab_visit_id
),
pocket_events AS (
SELECT
mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position,
mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id,
mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id,
COUNTIF(
event_name = 'save'
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS pocket_saves,
COUNTIF(
event_name = 'click'
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS pocket_clicks,
COUNTIF(
event_name = 'impression'
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS pocket_impressions,
COUNTIF(
event_name = 'click'
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS sponsored_pocket_clicks,
COUNTIF(
event_name = 'click'
AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS organic_pocket_clicks,
COUNTIF(
event_name = 'impression'
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS sponsored_pocket_impressions,
COUNTIF(
event_name = 'impression'
AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS organic_pocket_impressions,
COUNTIF(
event_name = 'save'
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS sponsored_pocket_saves,
COUNTIF(
event_name = 'save'
AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS organic_pocket_saves,
COUNTIF(
event_name = 'dismiss'
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS sponsored_pocket_dismissals,
COUNTIF(
event_name = 'dismiss'
AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
) AS organic_pocket_dismissals,
COUNTIF(
event_name = 'thumb_voting_interaction'
AND mozfun.map.get_key(event_details, "thumbs_up") = "true"
) AS pocket_thumbs_up,
COUNTIF(
event_name = 'thumb_voting_interaction'
AND mozfun.map.get_key(event_details, "thumbs_down") = "true"
) AS pocket_thumbs_down,
SAFE_CAST(mozfun.map.get_key(event_details, "received_rank") AS INT) AS pocket_received_rank,
mozfun.map.get_key(
event_details,
"scheduled_corpus_item_id"
) AS pocket_scheduled_corpus_item_id,
mozfun.map.get_key(event_details, "topic") AS pocket_topic,
mozfun.map.get_key(event_details, "matches_selected_topic") AS pocket_matches_selected_topic,
COUNTIF(
event_name = "click"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
) AS list_card_clicks,
COUNTIF(
event_name = "click"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
) AS organic_list_card_clicks,
COUNTIF(
event_name = "click"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
) AS sponsored_list_card_clicks,
COUNTIF(
event_name = "impression"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
) AS list_card_impressions,
COUNTIF(
event_name = "impression"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
) AS organic_list_card_impressions,
COUNTIF(
event_name = "impression"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
) AS sponsored_list_card_impressions,
COUNTIF(
event_name = "save"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
) AS list_card_saves,
COUNTIF(
event_name = "save"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
) AS organic_list_card_saves,
COUNTIF(
event_name = "save"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
) AS sponsored_list_card_saves,
COUNTIF(
event_name = "dismiss"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
) AS list_card_dismissals,
COUNTIF(
event_name = "dismiss"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
) AS organic_list_card_dismissals,
COUNTIF(
event_name = "dismiss"
AND mozfun.map.get_key(event_details, "is_list_card") = "true"
AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
) AS sponsored_list_card_dismissals,
FROM
events_unnested
WHERE
event_category = 'pocket'
AND event_name IN ('impression', 'click', 'save', 'dismiss', 'thumb_voting_interaction')
GROUP BY
newtab_visit_id,
pocket_story_position,
pocket_tile_id,
pocket_recommendation_id,
pocket_received_rank,
pocket_scheduled_corpus_item_id,
pocket_topic,
pocket_matches_selected_topic
),
pocket_summary AS (
SELECT
newtab_visit_id,
ARRAY_AGG(
STRUCT(
pocket_story_position,
pocket_tile_id,
pocket_recommendation_id,
pocket_impressions,
sponsored_pocket_impressions,
organic_pocket_impressions,
pocket_clicks,
sponsored_pocket_clicks,
organic_pocket_clicks,
pocket_saves,
sponsored_pocket_saves,
organic_pocket_saves,
sponsored_pocket_dismissals,
organic_pocket_dismissals,
pocket_thumbs_up,
pocket_thumbs_down,
pocket_received_rank,
pocket_scheduled_corpus_item_id,
pocket_topic,
pocket_matches_selected_topic,
list_card_clicks,
organic_list_card_clicks,
sponsored_list_card_clicks,
list_card_impressions,
organic_list_card_impressions,
sponsored_list_card_impressions,
list_card_saves,
organic_list_card_saves,
sponsored_list_card_saves,
list_card_dismissals,
organic_list_card_dismissals,
sponsored_list_card_dismissals
)
) AS pocket_interactions
FROM
pocket_events
GROUP BY
newtab_visit_id
),
wallpaper_events AS (
SELECT
mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper,
COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks,
COUNTIF(
event_name = 'wallpaper_click'
AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true"
) AS wallpaper_clicks_had_previous_wallpaper,
COUNTIF(
event_name = 'wallpaper_click'
AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false"
) AS wallpaper_clicks_first_selected_wallpaper,
COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks,
COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals,
COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks
FROM
events_unnested
WHERE
event_category = 'newtab'
AND event_name IN (
'wallpaper_click',
'wallpaper_category_click',
'wallpaper_highlight_cta_clicks',
'wallpaper_highlight_dismissed'
)
GROUP BY
newtab_visit_id,
wallpaper_selected_wallpaper
),
wallpaper_summary AS (
SELECT
newtab_visit_id,
ARRAY_AGG(
STRUCT(
wallpaper_selected_wallpaper,
wallpaper_clicks,
wallpaper_clicks_had_previous_wallpaper,
wallpaper_clicks_first_selected_wallpaper,
wallpaper_category_clicks,
wallpaper_highlight_dismissals,
wallpaper_highlight_cta_clicks
)
) AS wallpaper_interactions
FROM
wallpaper_events
GROUP BY
newtab_visit_id
),
weather_events AS (
SELECT
mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions,
COUNTIF(event_name = 'weather_location_selected') AS weather_widget_location_selected,
COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks,
COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors,
COUNTIF(
event_name = 'weather_change_display'
AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed"
) AS weather_widget_change_display_to_detailed,
COUNTIF(
event_name = 'weather_change_display'
AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple"
) AS weather_widget_change_display_to_simple
FROM
events_unnested
WHERE
event_category = 'newtab'
AND event_name IN (
'weather_impression',
'weather_open_provider_url',
'weather_load_error',
'weather_change_display',
'weather_location_selected'
)
GROUP BY
newtab_visit_id
),
weather_summary AS (
SELECT
newtab_visit_id,
ARRAY_AGG(
STRUCT(
weather_widget_impressions,
weather_widget_location_selected,
weather_widget_clicks,
weather_widget_load_errors,
weather_widget_change_display_to_detailed,
weather_widget_change_display_to_simple
)
) AS weather_interactions
FROM
weather_events
GROUP BY
newtab_visit_id
),
topic_selection_events AS (
SELECT
mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
mozfun.map.get_key(event_details, "previous_topics") AS previous_topics,
mozfun.map.get_key(event_details, "topics") AS topics,
COUNTIF(event_name = 'topic_selection_open') AS topic_selection_open,
COUNTIF(event_name = 'topic_selection_dismiss') AS topic_selection_dismiss,
COUNTIF(
event_name = 'topic_selection_topics_saved'
AND mozfun.map.get_key(event_details, "first_save") = "true"
) AS topic_selection_topics_first_saved,
COUNTIF(
event_name = 'topic_selection_topics_saved'
AND mozfun.map.get_key(event_details, "first_save") != "true"
) AS topic_selection_topics_updated,
FROM
events_unnested
WHERE
event_category = 'newtab'
AND event_name IN (
'topic_selection_dismiss',
'topic_selection_open',
'topic_selection_topics_saved'
)
GROUP BY
newtab_visit_id,
previous_topics,
topics
),
topic_selection_summary AS (
SELECT
newtab_visit_id,
ARRAY_AGG(
STRUCT(
previous_topics,
topics,
topic_selection_open,
topic_selection_dismiss,
topic_selection_topics_first_saved,
topic_selection_topics_updated
)
) AS topic_selection_interactions
FROM
topic_selection_events
GROUP BY
newtab_visit_id
),
combined_newtab_activity AS (
SELECT
*
FROM
visit_metadata
LEFT JOIN
search_summary
USING (newtab_visit_id)
LEFT JOIN
topsites_summary
USING (newtab_visit_id)
LEFT JOIN
pocket_summary
USING (newtab_visit_id)
LEFT JOIN
wallpaper_summary
USING (newtab_visit_id)
LEFT JOIN
weather_summary
USING (newtab_visit_id)
LEFT JOIN
topic_selection_summary
USING (newtab_visit_id)
WHERE
-- Keep only rows with interactions, unless we receive a valid newtab.opened event.
-- This is meant to drop only interactions that only have a newtab.closed event on the same partition
-- (these are suspected to be from pre-loaded tabs)
newtab_open_source IS NOT NULL
OR search_interactions IS NOT NULL
OR topsite_tile_interactions IS NOT NULL
OR pocket_interactions IS NOT NULL
OR wallpaper_interactions IS NOT NULL
OR weather_interactions IS NOT NULL
OR topic_selection_interactions IS NOT NULL
),
client_profile_info AS (
SELECT
client_id AS legacy_telemetry_client_id,
first_seen_date = @submission_date AS is_new_profile,
activity_segment
FROM
`moz-fx-data-shared-prod.telemetry.desktop_active_users`
WHERE
submission_date = @submission_date
)
SELECT
*,
CASE
WHEN (
(newtab_open_source = "about:home" AND newtab_homepage_category = "enabled")
OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled")
)
THEN "default"
ELSE "non-default"
END AS newtab_default_ui,
FROM
combined_newtab_activity
LEFT JOIN
client_profile_info
USING (legacy_telemetry_client_id)