sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v2/query.sql (568 lines of code) (raw):

-- Note that this query runs in the telemetry_derived dataset, so sees derived tables -- rather than the user-facing views (so key_value structs haven't been eliminated, etc.) WITH _current AS ( SELECT -- In this raw table, we capture the history of activity over the past -- 28 days for each usage criterion as a single 64-bit integer. The -- rightmost bit represents whether the user was active in the current day. CAST(TRUE AS INT64) AS days_seen_bits, CAST(active_hours_sum > 0 AS INT64) & CAST( COALESCE( scalar_parent_browser_engagement_total_uri_count_normal_and_private_mode_sum, scalar_parent_browser_engagement_total_uri_count_sum ) > 0 AS INT64 ) AS days_active_bits, -- For measuring Active MAU, where this is the days since this -- client_id was an Active User as defined by -- https://docs.telemetry.mozilla.org/cookbooks/active_dau.html CAST( COALESCE( scalar_parent_browser_engagement_total_uri_count_normal_and_private_mode_sum, scalar_parent_browser_engagement_total_uri_count_sum ) >= 1 AS INT64 ) AS days_visited_1_uri_bits, CAST( COALESCE( scalar_parent_browser_engagement_total_uri_count_normal_and_private_mode_sum, scalar_parent_browser_engagement_total_uri_count_sum ) >= 5 AS INT64 ) AS days_visited_5_uri_bits, CAST( COALESCE( scalar_parent_browser_engagement_total_uri_count_normal_and_private_mode_sum, scalar_parent_browser_engagement_total_uri_count_sum ) >= 10 AS INT64 ) AS days_visited_10_uri_bits, CAST(active_hours_sum >= 0.011 AS INT64) AS days_had_8_active_ticks_bits, CAST(devtools_toolbox_opened_count_sum > 0 AS INT64) AS days_opened_dev_tools_bits, CAST(active_hours_sum > 0 AS INT64) AS days_interacted_bits, CAST( scalar_parent_browser_engagement_total_uri_count_sum >= 1 AS INT64 ) AS days_visited_1_uri_normal_mode_bits, -- This field is only available after version 84, see the definition in clients_daily_v6 view CAST( IF( mozfun.norm.extract_version(app_display_version, 'major') < 84, NULL, scalar_parent_browser_engagement_total_uri_count_normal_and_private_mode_sum - COALESCE( scalar_parent_browser_engagement_total_uri_count_sum, 0 ) ) >= 1 AS INT64 ) AS days_visited_1_uri_private_mode_bits, -- We only trust profile_date if it is within one week of the ping submission, -- so we ignore any value more than seven days old. `moz-fx-data-shared-prod.udf.days_since_created_profile_as_28_bits`( DATE_DIFF(submission_date, SAFE.PARSE_DATE("%F", SUBSTR(profile_creation_date, 0, 10)), DAY) ) AS days_created_profile_bits, -- Experiments are an array, so we keep track of a usage bit pattern per experiment. ARRAY( SELECT AS STRUCT key AS experiment, value AS branch, 1 AS bits FROM UNNEST(experiments) ) AS days_seen_in_experiment, * EXCEPT (submission_date) FROM `moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6` WHERE submission_date = @submission_date ), -- _previous AS ( SELECT days_seen_bits, days_active_bits, days_visited_1_uri_bits, days_visited_5_uri_bits, days_visited_10_uri_bits, days_had_8_active_ticks_bits, days_opened_dev_tools_bits, days_interacted_bits, days_visited_1_uri_normal_mode_bits, days_visited_1_uri_private_mode_bits, days_created_profile_bits, days_seen_in_experiment, * EXCEPT ( days_seen_bits, days_active_bits, days_visited_1_uri_bits, days_visited_5_uri_bits, days_visited_10_uri_bits, days_had_8_active_ticks_bits, days_opened_dev_tools_bits, days_interacted_bits, days_visited_1_uri_normal_mode_bits, days_visited_1_uri_private_mode_bits, days_created_profile_bits, days_seen_in_experiment, submission_date, first_seen_date, second_seen_date ) FROM `moz-fx-data-shared-prod.telemetry_derived.clients_last_seen_v2` WHERE submission_date = DATE_SUB(@submission_date, INTERVAL 1 DAY) -- Filter out rows from yesterday that have now fallen outside the 28-day window. AND `moz-fx-data-shared-prod.udf.shift_28_bits_one_day`(days_seen_bits) > 0 ), staging AS ( SELECT @submission_date AS submission_date, IF(cfs.first_seen_date > @submission_date, NULL, cfs.first_seen_date) AS first_seen_date, IF(cfs.second_seen_date > @submission_date, NULL, cfs.second_seen_date) AS second_seen_date, IF(_current.client_id IS NOT NULL, _current, _previous).* REPLACE ( `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_seen_bits, _current.days_seen_bits ) AS days_seen_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_active_bits, _current.days_active_bits ) AS days_active_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_visited_1_uri_bits, _current.days_visited_1_uri_bits ) AS days_visited_1_uri_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_visited_5_uri_bits, _current.days_visited_5_uri_bits ) AS days_visited_5_uri_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_visited_10_uri_bits, _current.days_visited_10_uri_bits ) AS days_visited_10_uri_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_had_8_active_ticks_bits, _current.days_had_8_active_ticks_bits ) AS days_had_8_active_ticks_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_opened_dev_tools_bits, _current.days_opened_dev_tools_bits ) AS days_opened_dev_tools_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_interacted_bits, _current.days_interacted_bits ) AS days_interacted_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_visited_1_uri_normal_mode_bits, _current.days_visited_1_uri_normal_mode_bits ) AS days_visited_1_uri_normal_mode_bits, `moz-fx-data-shared-prod.udf.combine_adjacent_days_28_bits`( _previous.days_visited_1_uri_private_mode_bits, _current.days_visited_1_uri_private_mode_bits ) AS days_visited_1_uri_private_mode_bits, `moz-fx-data-shared-prod.udf.coalesce_adjacent_days_28_bits`( _previous.days_created_profile_bits, _current.days_created_profile_bits ) AS days_created_profile_bits, `moz-fx-data-shared-prod.udf.combine_experiment_days`( _previous.days_seen_in_experiment, _current.days_seen_in_experiment ) AS days_seen_in_experiment ) FROM _current FULL JOIN _previous USING (client_id) LEFT JOIN `moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v3` AS cfs USING (client_id) ) SELECT submission_date, first_seen_date, second_seen_date, days_seen_bits, days_visited_1_uri_bits, days_visited_5_uri_bits, days_visited_10_uri_bits, days_had_8_active_ticks_bits, days_opened_dev_tools_bits, days_interacted_bits, days_visited_1_uri_normal_mode_bits, days_visited_1_uri_private_mode_bits, days_created_profile_bits, days_seen_in_experiment, client_id, aborts_content_sum, aborts_gmplugin_sum, aborts_plugin_sum, active_addons_count_mean, active_addons, active_experiment_branch, active_experiment_id, active_hours_sum, addon_compatibility_check_enabled, app_build_id, app_display_version, app_name, app_version, attribution, blocklist_enabled, channel, client_clock_skew_mean, client_submission_latency_mean, cpu_cores, cpu_count, cpu_family, cpu_l2_cache_kb, cpu_l3_cache_kb, cpu_model, cpu_speed_mhz, cpu_stepping, cpu_vendor, crashes_detected_content_sum, crashes_detected_gmplugin_sum, crashes_detected_plugin_sum, crash_submit_attempt_content_sum, crash_submit_attempt_main_sum, crash_submit_attempt_plugin_sum, crash_submit_success_content_sum, crash_submit_success_main_sum, crash_submit_success_plugin_sum, default_search_engine, default_search_engine_data_load_path, default_search_engine_data_name, default_search_engine_data_origin, default_search_engine_data_submission_url, devtools_toolbox_opened_count_sum, distribution_id, e10s_enabled, env_build_arch, env_build_id, env_build_version, environment_settings_intl_accept_languages, environment_settings_intl_app_locales, environment_settings_intl_available_locales, environment_settings_intl_requested_locales, environment_settings_intl_system_locales, environment_settings_intl_regional_prefs_locales, experiments, first_paint_mean, flash_version, country, city, geo_subdivision1, geo_subdivision2, isp_name, isp_organization, gfx_features_advanced_layers_status, gfx_features_d2d_status, gfx_features_d3d11_status, gfx_features_gpu_process_status, histogram_parent_devtools_aboutdebugging_opened_count_sum, histogram_parent_devtools_animationinspector_opened_count_sum, histogram_parent_devtools_browserconsole_opened_count_sum, histogram_parent_devtools_canvasdebugger_opened_count_sum, histogram_parent_devtools_computedview_opened_count_sum, histogram_parent_devtools_custom_opened_count_sum, histogram_parent_devtools_developertoolbar_opened_count_sum, histogram_parent_devtools_dom_opened_count_sum, histogram_parent_devtools_eyedropper_opened_count_sum, histogram_parent_devtools_fontinspector_opened_count_sum, histogram_parent_devtools_inspector_opened_count_sum, histogram_parent_devtools_jsbrowserdebugger_opened_count_sum, histogram_parent_devtools_jsdebugger_opened_count_sum, histogram_parent_devtools_jsprofiler_opened_count_sum, histogram_parent_devtools_layoutview_opened_count_sum, histogram_parent_devtools_memory_opened_count_sum, histogram_parent_devtools_menu_eyedropper_opened_count_sum, histogram_parent_devtools_netmonitor_opened_count_sum, histogram_parent_devtools_options_opened_count_sum, histogram_parent_devtools_paintflashing_opened_count_sum, histogram_parent_devtools_picker_eyedropper_opened_count_sum, histogram_parent_devtools_responsive_opened_count_sum, histogram_parent_devtools_ruleview_opened_count_sum, histogram_parent_devtools_scratchpad_opened_count_sum, histogram_parent_devtools_scratchpad_window_opened_count_sum, histogram_parent_devtools_shadereditor_opened_count_sum, histogram_parent_devtools_storage_opened_count_sum, histogram_parent_devtools_styleeditor_opened_count_sum, histogram_parent_devtools_webaudioeditor_opened_count_sum, histogram_parent_devtools_webconsole_opened_count_sum, histogram_parent_devtools_webide_opened_count_sum, install_year, is_default_browser, is_wow64, locale, memory_mb, normalized_channel, normalized_os_version, os, os_service_pack_major, os_service_pack_minor, os_version, pings_aggregated_by_this_row, places_bookmarks_count_mean, places_pages_count_mean, plugin_hangs_sum, plugins_infobar_allow_sum, plugins_infobar_block_sum, plugins_infobar_shown_sum, plugins_notification_shown_sum, previous_build_id, profile_age_in_days, profile_creation_date, push_api_notify_sum, sample_id, sandbox_effective_content_process_level, scalar_combined_webrtc_nicer_stun_retransmits_sum, scalar_combined_webrtc_nicer_turn_401s_sum, scalar_combined_webrtc_nicer_turn_403s_sum, scalar_combined_webrtc_nicer_turn_438s_sum, scalar_content_navigator_storage_estimate_count_sum, scalar_content_navigator_storage_persist_count_sum, scalar_parent_aushelper_websense_reg_version, scalar_parent_browser_engagement_max_concurrent_tab_count_max, scalar_parent_browser_engagement_max_concurrent_window_count_max, scalar_parent_browser_engagement_tab_open_event_count_sum, scalar_parent_browser_engagement_total_uri_count_sum, scalar_parent_browser_engagement_unfiltered_uri_count_sum, scalar_parent_browser_engagement_unique_domains_count_max, scalar_parent_browser_engagement_unique_domains_count_mean, scalar_parent_browser_engagement_window_open_event_count_sum, scalar_parent_devtools_accessibility_node_inspected_count_sum, scalar_parent_devtools_accessibility_opened_count_sum, scalar_parent_devtools_accessibility_picker_used_count_sum, scalar_parent_devtools_accessibility_select_accessible_for_node_sum, scalar_parent_devtools_accessibility_service_enabled_count_sum, scalar_parent_devtools_copy_full_css_selector_opened_sum, scalar_parent_devtools_copy_unique_css_selector_opened_sum, scalar_parent_devtools_toolbar_eyedropper_opened_sum, scalar_parent_dom_contentprocess_troubled_due_to_memory_sum, scalar_parent_navigator_storage_estimate_count_sum, scalar_parent_navigator_storage_persist_count_sum, scalar_parent_storage_sync_api_usage_extensions_using_sum, search_cohort, search_count_abouthome, search_count_contextmenu, search_count_newtab, search_count_searchbar, search_count_system, search_count_urlbar, search_count_all, search_count_tagged_sap, search_count_tagged_follow_on, search_count_organic, search_count_urlbar_handoff, session_restored_mean, sessions_started_on_this_day, shutdown_kill_sum, subsession_hours_sum, ssl_handshake_result_failure_sum, ssl_handshake_result_success_sum, sync_configured, sync_count_desktop_mean, sync_count_mobile_mean, sync_count_desktop_sum, sync_count_mobile_sum, telemetry_enabled, timezone_offset, total_hours_sum, update_auto_download, update_channel, update_enabled, vendor, web_notification_shown_sum, windows_build_number, windows_ubr, fxa_configured, trackers_blocked_sum, submission_timestamp_min, ad_clicks_count_all, search_with_ads_count_all, scalar_parent_urlbar_impression_autofill_about_sum, scalar_parent_urlbar_impression_autofill_adaptive_sum, scalar_parent_urlbar_impression_autofill_origin_sum, scalar_parent_urlbar_impression_autofill_other_sum, scalar_parent_urlbar_impression_autofill_preloaded_sum, scalar_parent_urlbar_impression_autofill_url_sum, scalar_parent_telemetry_event_counts_sum, scalar_content_telemetry_event_counts_sum, scalar_parent_urlbar_searchmode_bookmarkmenu_sum, scalar_parent_urlbar_searchmode_handoff_sum, scalar_parent_urlbar_searchmode_keywordoffer_sum, scalar_parent_urlbar_searchmode_oneoff_sum, scalar_parent_urlbar_searchmode_other_sum, scalar_parent_urlbar_searchmode_shortcut_sum, scalar_parent_urlbar_searchmode_tabmenu_sum, scalar_parent_urlbar_searchmode_tabtosearch_sum, scalar_parent_urlbar_searchmode_tabtosearch_onboard_sum, scalar_parent_urlbar_searchmode_topsites_newtab_sum, scalar_parent_urlbar_searchmode_topsites_urlbar_sum, scalar_parent_urlbar_searchmode_touchbar_sum, scalar_parent_urlbar_searchmode_typed_sum, scalar_parent_os_environment_is_taskbar_pinned, scalar_parent_os_environment_launched_via_desktop, scalar_parent_os_environment_launched_via_start_menu, scalar_parent_os_environment_launched_via_taskbar, scalar_parent_os_environment_launched_via_other_shortcut, scalar_parent_os_environment_launched_via_other, search_count_webextension, search_count_alias, search_count_urlbar_searchmode, scalar_parent_browser_ui_interaction_preferences_pane_home_sum, scalar_parent_urlbar_picked_autofill_sum, scalar_parent_urlbar_picked_autofill_about_sum, scalar_parent_urlbar_picked_autofill_adaptive_sum, scalar_parent_urlbar_picked_autofill_origin_sum, scalar_parent_urlbar_picked_autofill_other_sum, scalar_parent_urlbar_picked_autofill_preloaded_sum, scalar_parent_urlbar_picked_autofill_url_sum, scalar_parent_urlbar_picked_bookmark_sum, scalar_parent_urlbar_picked_dynamic_sum, scalar_parent_urlbar_picked_extension_sum, scalar_parent_urlbar_picked_formhistory_sum, scalar_parent_urlbar_picked_history_sum, scalar_parent_urlbar_picked_keyword_sum, scalar_parent_urlbar_picked_remotetab_sum, scalar_parent_urlbar_picked_searchengine_sum, scalar_parent_urlbar_picked_searchsuggestion_sum, scalar_parent_urlbar_picked_switchtab_sum, scalar_parent_urlbar_picked_tabtosearch_sum, scalar_parent_urlbar_picked_tip_sum, scalar_parent_urlbar_picked_topsite_sum, scalar_parent_urlbar_picked_unknown_sum, scalar_parent_urlbar_picked_visiturl_sum, default_private_search_engine, default_private_search_engine_data_load_path, default_private_search_engine_data_name, default_private_search_engine_data_origin, default_private_search_engine_data_submission_url, search_counts, user_pref_browser_search_region, search_with_ads, ad_clicks, search_content_urlbar_sum, search_content_urlbar_handoff_sum, search_content_urlbar_searchmode_sum, search_content_contextmenu_sum, search_content_about_home_sum, search_content_about_newtab_sum, search_content_searchbar_sum, search_content_system_sum, search_content_webextension_sum, search_content_tabhistory_sum, search_content_reload_sum, search_content_unknown_sum, search_withads_urlbar_sum, search_withads_urlbar_handoff_sum, search_withads_urlbar_searchmode_sum, search_withads_contextmenu_sum, search_withads_about_home_sum, search_withads_about_newtab_sum, search_withads_searchbar_sum, search_withads_system_sum, search_withads_webextension_sum, search_withads_tabhistory_sum, search_withads_reload_sum, search_withads_unknown_sum, search_adclicks_urlbar_sum, search_adclicks_urlbar_handoff_sum, search_adclicks_urlbar_searchmode_sum, search_adclicks_contextmenu_sum, search_adclicks_about_home_sum, search_adclicks_about_newtab_sum, search_adclicks_searchbar_sum, search_adclicks_system_sum, search_adclicks_webextension_sum, search_adclicks_tabhistory_sum, search_adclicks_reload_sum, search_adclicks_unknown_sum, update_background, user_pref_browser_search_suggest_enabled, user_pref_browser_widget_in_navbar, user_pref_browser_urlbar_suggest_searches, user_pref_browser_urlbar_show_search_suggestions_first, user_pref_browser_urlbar_suggest_quicksuggest, user_pref_browser_urlbar_suggest_quicksuggest_sponsored, user_pref_browser_urlbar_quicksuggest_onboarding_dialog_choice, scalar_parent_browser_engagement_total_uri_count_normal_and_private_mode_sum, user_pref_browser_newtabpage_enabled, user_pref_app_shield_optoutstudies_enabled, contextual_services_quicksuggest_click_sum, contextual_services_quicksuggest_impression_sum, contextual_services_quicksuggest_help_sum, contextual_services_topsites_click_sum, contextual_services_topsites_impression_sum, user_pref_browser_urlbar_suggest_quicksuggest_nonsponsored, user_pref_browser_urlbar_quicksuggest_data_collection_enabled, scalar_a11y_hcm_foreground, scalar_a11y_hcm_background, a11y_theme, contextual_services_quicksuggest_help_nonsponsored_bestmatch_sum, contextual_services_quicksuggest_help_sponsored_bestmatch_sum, contextual_services_quicksuggest_block_nonsponsored_sum, contextual_services_quicksuggest_block_sponsored_sum, contextual_services_quicksuggest_block_sponsored_bestmatch_sum, contextual_services_quicksuggest_block_nonsponsored_bestmatch_sum, contextual_services_quicksuggest_click_sponsored_bestmatch_sum, contextual_services_quicksuggest_click_nonsponsored_bestmatch_sum, contextual_services_quicksuggest_impression_sponsored_bestmatch_sum, contextual_services_quicksuggest_impression_nonsponsored_bestmatch_sum, user_pref_browser_urlbar_suggest_bestmatch, scalar_parent_browser_ui_interaction_textrecognition_error_sum, text_recognition_interaction_timing_sum, text_recognition_interaction_timing_count_sum, scalar_parent_browser_ui_interaction_content_context_sum, text_recognition_api_performance_sum, text_recognition_api_performance_count_sum, text_recognition_text_length_sum, text_recognition_text_length_count_sum, scalar_parent_os_environment_launched_via_taskbar_private, dom_parentprocess_private_window_used, os_environment_is_taskbar_pinned_any, os_environment_is_taskbar_pinned_private_any, os_environment_is_taskbar_pinned_private, bookmark_migrations_quantity_chrome, bookmark_migrations_quantity_edge, bookmark_migrations_quantity_safari, bookmark_migrations_quantity_all, history_migrations_quantity_chrome, history_migrations_quantity_edge, history_migrations_quantity_safari, history_migrations_quantity_all, logins_migrations_quantity_chrome, logins_migrations_quantity_edge, logins_migrations_quantity_safari, logins_migrations_quantity_all, search_count_urlbar_persisted, search_content_urlbar_persisted_sum, search_withads_urlbar_persisted_sum, search_adclicks_urlbar_persisted_sum, media_play_time_ms_audio_sum, media_play_time_ms_video_sum, contextual_services_quicksuggest_block_dynamic_wikipedia_sum, contextual_services_quicksuggest_block_weather_sum, contextual_services_quicksuggest_click_dynamic_wikipedia_sum, contextual_services_quicksuggest_click_nonsponsored_sum, contextual_services_quicksuggest_click_sponsored_sum, contextual_services_quicksuggest_click_weather_sum, contextual_services_quicksuggest_help_dynamic_wikipedia_sum, contextual_services_quicksuggest_help_nonsponsored_sum, contextual_services_quicksuggest_help_sponsored_sum, contextual_services_quicksuggest_help_weather_sum, contextual_services_quicksuggest_impression_dynamic_wikipedia_sum, contextual_services_quicksuggest_impression_nonsponsored_sum, contextual_services_quicksuggest_impression_sponsored_sum, contextual_services_quicksuggest_impression_weather_sum, places_searchbar_cumulative_searches_sum, places_searchbar_cumulative_filter_count_sum, scalar_parent_sidebar_opened_sum, scalar_parent_sidebar_search_sum, scalar_parent_sidebar_link_sum, places_previousday_visits_mean, places_library_cumulative_bookmark_searches_sum, places_library_cumulative_history_searches_sum, places_bookmarks_searchbar_cumulative_searches_sum, scalar_parent_library_link_sum, scalar_parent_library_opened_sum, scalar_parent_library_search_sum, startup_profile_selection_reason_first, first_document_id, partner_id, distribution_version, distributor, distributor_channel, env_build_platform_version, env_build_xpcom_abi, geo_db_version, apple_model_id, max_subsession_counter, min_subsession_counter, startup_profile_selection_first_ping_only, days_active_bits, profile_group_id FROM staging a