components/llm_service/notebooks/VertexSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a588966e-ca3c-4556-93b7-b83b9a5e6cdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "PROJECT_ID = \"nasa-genie-dev\"\n",
    "REGION = \"us-central1\"\n",
    "import os\n",
    "os.environ[\"PROJECT_ID\"] = PROJECT_ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "44fe5e42-8889-414d-95b1-bde787b2c6c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../common/src\")\n",
    "sys.path.append(\"../src\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7aacbd5d-e2a0-4cb4-ae20-dd746f185b84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: [config/config.py:57 - <module>()] Namespace File not found, setting job namespace as default\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:173: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatDatabricks`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:343: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatMlflow`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:173: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatDatabricks`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:343: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatMlflow`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:173: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatDatabricks`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:343: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatMlflow`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:173: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatDatabricks`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:343: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatMlflow`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:173: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatDatabricks`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:343: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatMlflow`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:173: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatDatabricks`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:343: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatMlflow`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:173: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatDatabricks`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:343: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatMlflow`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.chat_models.cohere.ChatCohere` was deprecated in langchain-community 0.0.30 and will be removed in 0.2.0. An updated version of the class exists in the langchain-cohere package and should be used instead. To use it run `pip install -U langchain-cohere` and import as `from langchain_cohere import ChatCohere`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:173: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatDatabricks`.\n",
      "  warn_deprecated(\n",
      "/Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/.venv/lib/python3.9/site-packages/langchain_community/llms/__init__.py:343: LangChainDeprecationWarning: `` was deprecated in LangChain 0.0.22 and will be removed in 0.2. An updated version of the  exists in the langchain-community package and should be used instead. To use it run `pip install -U langchain-community` and import as `from langchain_community.chat_models import ChatMlflow`.\n",
      "  warn_deprecated(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: [config/config.py:107 - <module>()] ENABLE_GOOGLE_LLM = True\n",
      "INFO: [config/config.py:108 - <module>()] ENABLE_OPENAI_LLM = True\n",
      "INFO: [config/config.py:109 - <module>()] ENABLE_COHERE_LLM = True\n",
      "INFO: [config/config.py:110 - <module>()] ENABLE_GOOGLE_MODEL_GARDEN = True\n",
      "INFO: [config/config.py:111 - <module>()] ENABLE_TRUSS_LLAMA2 = True\n",
      "INFO: [config/vector_store_config.py:40 - <module>()] Default vector store = [matching_engine]\n",
      "INFO: [config/vector_store_config.py:49 - <module>()] PG_HOST = [10.10.0.2]\n",
      "INFO: [config/vector_store_config.py:50 - <module>()] PG_DBNAME = [pgvector]\n",
      "ERROR: [config/vector_store_config.py:77 - <module>()] Cannot connect to pgvector instance at 10.10.0.2: (psycopg2.OperationalError) connection to server at \"10.10.0.2\", port 5432 failed: Network is unreachable\n",
      "\tIs the server running on that host and accepting TCP/IP connections?\n",
      "\n",
      "(Background on this error at: https://sqlalche.me/e/20/e3q8)\n",
      "INFO: [config/onedrive_config.py:30 - <module>()] ONEDRIVE_CLIENT_ID = [None]\n",
      "INFO: [config/onedrive_config.py:31 - <module>()] ONEDRIVE_TENANT_ID = [None]\n",
      "INFO: [utils/text_helper.py:36 - <module>()] loaded spacy model\n"
     ]
    }
   ],
   "source": [
    "import tempfile\n",
    "import os\n",
    "from typing import List, Optional, Tuple, Dict\n",
    "from common.utils.logging_handler import Logger\n",
    "from common.models import (UserQuery, QueryResult, QueryEngine, QueryDocument,\n",
    "                           QueryReference, QueryDocumentChunk, BatchJobModel)\n",
    "from common.utils.errors import (ResourceNotFoundException,\n",
    "                                 ValidationError)\n",
    "from common.utils.http_exceptions import InternalServerError\n",
    "from utils.errors import NoDocumentsIndexedException\n",
    "from google.cloud import storage\n",
    "from services.query import query_prompts\n",
    "from services.query.vector_store import VectorStore\n",
    "from services.query.data_source import DataSource\n",
    "    \n",
    "from config import (PROJECT_ID, DEFAULT_QUERY_CHAT_MODEL,\n",
    "                        DEFAULT_QUERY_EMBEDDING_MODEL)\n",
    "from services.query.vertex_search import build_vertex_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3d8cd82f-4bac-4291-9db9-25ee3527b6d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "storage_client = storage.Client(project=PROJECT_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "07c2c33b-2ea6-4e17-8c56-4160a7e78e09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('nasa-search-vertex-v2', '6O2KmKchHgJUSkgrqARq'),\n",
       " ('nasa-search-integrated-v5', 'nqIU7Oi4qRIOgKllDqsY'),\n",
       " ('nasa-search-shpt-v5', 'bteIzSiaaCd3t8vjzDLw'),\n",
       " ('nasa-search-gcs-v5', 'FZnCyfyWNsk2gcHErMVR'),\n",
       " ('nasa-search-integrated-v4', 'LjUPZcMrUlo2KU78R8Qf'),\n",
       " ('nasa-search-gcs-v4', 'z9mUTW7JehYEvimZlL9p'),\n",
       " ('nasa-search-integrated-v3', 'Q6Pwa5VmIL89wSKcwoHQ'),\n",
       " ('nasa-search-shpt-v3', 'Qooyhj9PMWmFaiXbuT8E'),\n",
       " ('nasa-search-gcs-v3', 'FeY4jc6VCOADwQEXTSej'),\n",
       " ('nasa-search-shpt-v2', '4DTQXZfJzLai7tbh3iFX'),\n",
       " ('nasa-search-vertex-v1', 'BAlAKbfzfzB33PQQpJLW'),\n",
       " ('nasa-search-gcs-v2', 'tgqwj9ANxTY2Sd4iYdrg'),\n",
       " ('Nasa Blogs', 'kbWG1DkFPr7xsDUdkxcf'),\n",
       " ('nasa-search-integrated-v2', 'TEuxFc0HRTYGYUEmVV1y'),\n",
       " ('nasa-search-gcs', '6kcXnzQGxUkWHTcusn5j'),\n",
       " ('Vertex Search', 'YNaSypni55kAlF9xp8L8')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qe_list = QueryEngine.fetch_all()\n",
    "[(qe.name, qe.id) for qe in qe_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f6bb4216-da9e-43cb-8573-3694423d81df",
   "metadata": {},
   "outputs": [],
   "source": [
    "q_engine = QueryEngine.find_by_name('nasa-search-vertex-v2')\n",
    "\n",
    "QueryDocument.collection.filter(\n",
    "  \"query_engine\", \"==\", 'nasa-search-vertex-v2'\n",
    ").delete()\n",
    "\n",
    "QueryDocumentChunk.collection.filter(\n",
    "  \"query_engine_id\", \"==\", q_engine.id\n",
    ").delete()\n",
    "\n",
    "QueryReference.collection.filter(\n",
    "  \"query_engine\", \"==\", 'nasa-search-vertex-v2'\n",
    ").delete()\n",
    "\n",
    "QueryResult.collection.filter(\n",
    "  \"query_engine\", \"==\", 'nasa-search-vertex-v2'\n",
    ").delete()\n",
    "QueryEngine.delete_by_id(q_engine.id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "246c2f1a-94b0-4c21-b768-a28759d77a58",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine = \"Test Vertex Search 7\"\n",
    "doc_url = \"https://wisconsindot.gov/pages/online-srvcs/external/dmv.aspx\"\n",
    "user_id = \"epKVwPZeIBz1rrqcDm1B\"\n",
    "is_public=True\n",
    "query_description=\"test vertex search\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "87a5930c-55f3-4489-b589-c7127f2a7bdb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<common.models.llm_query.QueryEngine at 0x32887d2b0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from common.models.llm_query import QE_TYPE_VERTEX_SEARCH, QE_TYPE_LLM_SERVICE\n",
    "\n",
    "# create q_engine\n",
    "params = {\"depth_limit\": 1}\n",
    "query_engine_type = QE_TYPE_VERTEX_SEARCH\n",
    "embedding_type = DEFAULT_QUERY_EMBEDDING_MODEL\n",
    "llm_type = DEFAULT_QUERY_EMBEDDING_MODEL\n",
    "vector_store_type = None\n",
    "q_engine = QueryEngine(name=query_engine,\n",
    "                     created_by=user_id,\n",
    "                     query_engine_type=QE_TYPE_VERTEX_SEARCH,\n",
    "                     llm_type=llm_type,\n",
    "                     description=query_description,\n",
    "                     embedding_type=embedding_type,\n",
    "                     vector_store=vector_store_type,\n",
    "                     params=params,\n",
    "                     doc_url=doc_url)\n",
    "q_engine.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c1dae15e-da0e-485a-a7ff-3f9532dad267",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#build_vertex_search(q_engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6ef32db7-92fe-4c1e-9d0b-7a2506aa02c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import discoveryengine_v1alpha as discoveryengine\n",
    "location = \"global\"\n",
    "project_id = PROJECT_ID\n",
    "parent = \"projects/lramsey-dev/locations/global/collections/default_collection\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d65c780b-9a63-480d-8c93-221b04cdf5c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from services.query.vertex_search import (create_data_store, \n",
    "                                          create_search_engine, \n",
    "                                          import_documents_to_datastore,\n",
    "                                          download_web_docs,\n",
    "                                          import_documents_gcs, wait_for_operation, datastore_id_from_engine, inventory_gcs_files,\n",
    "                                          wait_for_operation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6e126185-173b-4861-ab72-e9f9083cbb53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'test-vertex-search-6'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_store_id = datastore_id_from_engine(q_engine)\n",
    "data_store_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "95a89e93-e3fa-47a0-b544-9c2a1a511000",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_url = q_engine.doc_url\n",
    "project_id = PROJECT_ID\n",
    "location = \"global\"\n",
    "\n",
    "docs_to_be_processed = []\n",
    "docs_processed = []\n",
    "docs_not_processed = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "db4b3d02-6ed2-4906-a538-8a3cf5de2ebb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: [query/vertex_search.py:490 - download_web_docs()] downloading web docs to bucket [lramsey-dev-downloads-test-vertex-search-6]\n",
      "INFO: [utils/gcs_helper.py:45 - create_bucket()] Bucket lramsey-dev-downloads-test-vertex-search-6 created.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-26 08:18:56 [scrapy.utils.log] INFO: Scrapy 2.11.0 started (bot: scrapybot)\n",
      "2024-02-26 08:18:56 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.9.13 (v3.9.13:6de2ca5339, May 17 2022, 11:37:23) - [Clang 13.0.0 (clang-1300.0.29.30)], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.2, Platform macOS-14.3.1-x86_64-i386-64bit\n",
      "2024-02-26 08:18:56 [scrapy.addons] INFO: Enabled addons:\n",
      "[]\n",
      "2024-02-26 08:18:56 [py.warnings] WARNING: /Users/lramsey/work/ailp/lukmanr-gps-core-solution-services/components/llm_service/notebooks/.venv/lib/python3.9/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n",
      "\n",
      "It is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n",
      "\n",
      "See the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n",
      "  return cls(crawler)\n",
      "\n",
      "2024-02-26 08:18:56 [scrapy.extensions.telnet] INFO: Telnet Password: a64840bb8e487ba2\n",
      "2024-02-26 08:18:56 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.memusage.MemoryUsage',\n",
      " 'scrapy.extensions.logstats.LogStats']\n",
      "2024-02-26 08:18:56 [scrapy.crawler] INFO: Overridden settings:\n",
      "{'DEPTH_LIMIT': 1, 'LOG_LEVEL': 'INFO'}\n",
      "2024-02-26 08:18:56 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n",
      "2024-02-26 08:18:56 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2024-02-26 08:18:56 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "[]\n",
      "2024-02-26 08:18:56 [scrapy.core.engine] INFO: Spider opened\n",
      "2024-02-26 08:18:56 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2024-02-26 08:18:56 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: [query/web_datasource.py:46 - save_content()] Saving pages_online-srvcs_external_dmv.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 182 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading pages_online-srvcs_external_dmv.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 182 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/pages/online-srvcs/external/dmv.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_cons-protect_buy-sell-veh_buyorsellavehicle.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 158 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_cons-protect_buy-sell-veh_buyorsellavehicle.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 158 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/cons-protect/buy-sell-veh/buyorsellavehicle.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_cons-protect_lemon-law_lemonlaw.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 146 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_cons-protect_lemon-law_lemonlaw.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 146 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/cons-protect/lemon-law/lemonlaw.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_agri-eq-veh_ioh.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 231 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_agri-eq-veh_ioh.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 231 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ioh.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_agri-eq-veh_ag-cmv.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 250 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_agri-eq-veh_ag-cmv.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 250 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-cmv.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_cons-protect_know-rghts_knowyourrights.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 153 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_cons-protect_know-rghts_knowyourrights.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 153 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/cons-protect/know-rghts/knowyourrights.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_agri-eq-veh_axle-weights.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 225 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_agri-eq-veh_axle-weights.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 225 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/agri-eq-veh/axle-weights.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_cons-protect_file-cmplnt_filedealercomplaint.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 170 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_cons-protect_file-cmplnt_filedealercomplaint.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 170 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/cons-protect/file-cmplnt/filedealercomplaint.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_agri-eq-veh_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 237 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_agri-eq-veh_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 237 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/agri-eq-veh/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_dlr-agents_dlr-training_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 232 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_dlr-agents_dlr-training_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 232 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-training/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_dlr-agents_dlr-inds-plt_dlrspecialplate.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 174 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_dlr-agents_dlr-inds-plt_dlrspecialplate.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 174 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-inds-plt/dlrspecialplate.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_dlr-agents_tips-tls-dlr_tips.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 163 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_dlr-agents_tips-tls-dlr_tips.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 163 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/dlr-agents/tips-tls-dlr/tips.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_dlr-agents_dlr-frms-pbs_dealer-forms.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 166 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_dlr-agents_dlr-frms-pbs_dealer-forms.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 166 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-frms-pbs/dealer-forms.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_dlr-agents_dlr-ttl-reg_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 166 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_dlr-agents_dlr-ttl-reg_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 166 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-ttl-reg/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_dlr-agents_busns-lcnse_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 168 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_dlr-agents_busns-lcnse_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 168 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/dlr-agents/busns-lcnse/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_dlr-agents_dlr-training_dealer.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 231 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_dlr-agents_dlr-training_dealer.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 231 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-training/dealer.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_global-footer_translate.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 211 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_global-footer_translate.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 211 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/global-footer/translate.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_careers_employees.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 234 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_careers_employees.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 234 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/careers/employees.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_global-footer_software.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_global-footer_software.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/global-footer/software.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_global-footer_Privacy-notice.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 216 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_global-footer_Privacy-notice.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 216 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/global-footer/Privacy-notice.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_global-footer_Legal-notices.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 215 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_global-footer_Legal-notices.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 215 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/global-footer/Legal-notices.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_accessibility-statement.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 225 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_accessibility-statement.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 225 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/accessibility-statement.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_global-footer_Acceptable-use-policy.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 223 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_global-footer_Acceptable-use-policy.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 223 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/global-footer/Acceptable-use-policy.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_global-footer_Support.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 209 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_global-footer_Support.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 209 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/global-footer/Support.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_global-footer_formdocs_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 207 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_global-footer_formdocs_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 207 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/global-footer/formdocs/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_title-plates_dmvagent.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 167 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_title-plates_dmvagent.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 167 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/dmvagent.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_contact-us_dmv-contact.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 148 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_contact-us_dmv-contact.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 148 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/contact-us/dmv-contact.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_frms-pubs_veh-forms.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 180 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_frms-pubs_veh-forms.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 180 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/frms-pubs/veh-forms.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_incidents.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 200 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_incidents.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 200 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/incidents.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_prkg-tckt_Web-TVRP.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 280 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_prkg-tckt_Web-TVRP.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 280 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/prkg-tckt/Web-TVRP.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_find-dmv_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 165 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_find-dmv_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 165 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/find-dmv/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_rcd-crsh-rpt_optout.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 144 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_rcd-crsh-rpt_optout.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 144 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/rcd-crsh-rpt/optout.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_rcd-crsh-rpt_driver-forms.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 177 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_rcd-crsh-rpt_driver-forms.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 177 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/rcd-crsh-rpt/driver-forms.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_req-veh-record.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 175 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_req-veh-record.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 175 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/req-veh-record.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_dsbld-prkg_temp-discard.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 175 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_dsbld-prkg_temp-discard.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 175 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/temp-discard.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_request-record.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 168 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_request-record.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 168 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/request-record.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_sll-jnk-vhcl_sellyourvehicle.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 154 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_sll-jnk-vhcl_sellyourvehicle.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 154 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/sll-jnk-vhcl/sellyourvehicle.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_dsbld-prkg_discards.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 156 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_dsbld-prkg_discards.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 156 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/discards.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_dsbld-prkg_displate.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 168 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_dsbld-prkg_displate.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 168 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/displate.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_title-plates_special-list.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 159 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_title-plates_special-list.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 159 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/special-list.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_rnew-plts_emissiontest.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 150 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_rnew-plts_emissiontest.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 150 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/rnew-plts/emissiontest.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_title-plates_nolaf.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 161 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_title-plates_nolaf.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 161 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/nolaf.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_dup-c-of-r.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 174 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_dup-c-of-r.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 174 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/dup-c-of-r.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_title-plates_new-res-default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 150 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_title-plates_new-res-default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 150 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/new-res-default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_renew-licens_plate-default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 165 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_renew-licens_plate-default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 165 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/renew-licens/plate-default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_title-plates_osow.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 163 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_title-plates_osow.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 163 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/osow.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_mtr-car-trkr_audit.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 170 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_mtr-car-trkr_audit.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 170 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/audit.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_mtr-car-trkr_irp.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 174 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_mtr-car-trkr_irp.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 174 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/irp.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_self-cert.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 200 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_self-cert.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 200 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/self-cert.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_mtr-car-trkr_mc-ins.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 160 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_mtr-car-trkr_mc-ins.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 160 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/mc-ins.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_cdl-how-aply_practiceapplication.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 157 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_cdl-how-aply_practiceapplication.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 157 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-how-aply/practiceapplication.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_rqts-for-cdl_fmcsa-reg-update-info.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 272 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_rqts-for-cdl_fmcsa-reg-update-info.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 272 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/fmcsa-reg-update-info.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_mtr-car-trkr_ifta.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 176 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_mtr-car-trkr_ifta.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 176 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/ifta.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_title-plates_walkin.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 184 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_title-plates_walkin.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 184 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/walkin.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving pages_dmv_license-drvs_rcd-crsh-rpt_auto-insurance.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 167 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading pages_dmv_license-drvs_rcd-crsh-rpt_auto-insurance.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 167 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/pages/dmv/license-drvs/rcd-crsh-rpt/auto-insurance.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_susp-or-rvkd_owi.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 172 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_susp-or-rvkd_owi.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 172 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/susp-or-rvkd/owi.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_how-to-apply_handbook.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 159 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_how-to-apply_handbook.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 159 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/handbook.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_rnew-and-chge_donor.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 159 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_rnew-and-chge_donor.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 159 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/donor.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_rnew-and-chge_veterans.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 154 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_rnew-and-chge_veterans.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 154 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/veterans.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_mdcl-cncrns_med-concerns.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 171 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_mdcl-cncrns_med-concerns.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 171 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/mdcl-cncrns/med-concerns.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_mdcl-cncrns_olderdrivers.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 152 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_mdcl-cncrns_olderdrivers.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 152 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/mdcl-cncrns/olderdrivers.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_how-to-apply_id-card.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 174 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_how-to-apply_id-card.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 174 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/id-card.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_rnew-and-chge_license-renewal.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 147 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_rnew-and-chge_license-renewal.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 147 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/license-renewal.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_motorcycles_mc-how-aply_schedulearoadtest.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 169 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_motorcycles_mc-how-aply_schedulearoadtest.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 169 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/motorcycles/mc-how-aply/schedulearoadtest.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_title-plates_status.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 169 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_title-plates_status.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 169 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/status.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_duplicate-id.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 175 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_duplicate-id.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 175 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/duplicate-id.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_how-to-apply_petition-process.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 191 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_how-to-apply_petition-process.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 191 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/petition-process.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_status-check.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 239 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_status-check.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 239 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/status-check.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_lst-or-stoln_duplicate-id.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 179 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_lst-or-stoln_duplicate-id.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 179 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/lst-or-stoln/duplicate-id.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_susp-or-rvkd_reinstate.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 165 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_susp-or-rvkd_reinstate.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 165 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/susp-or-rvkd/reinstate.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_rnew-and-chge_name-change.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 148 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_rnew-and-chge_name-change.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 148 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/name-change.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_change-addy_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 156 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_change-addy_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 156 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/change-addy/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_how-to-apply_get-lic.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 169 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_how-to-apply_get-lic.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 169 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/get-lic.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_teen-driver_teen-sfty_index.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 155 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_teen-driver_teen-sfty_index.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 155 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/teen-driver/teen-sfty/index.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_license-drvs_how-to-apply_roadtestgeneral.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 156 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_license-drvs_how-to-apply_roadtestgeneral.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 156 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/roadtestgeneral.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_duplicate-license.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 180 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_duplicate-license.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 180 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/duplicate-license.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_title-vehicle_emvpublicdefault.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 195 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_title-vehicle_emvpublicdefault.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 195 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/title-vehicle/emvpublicdefault.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_open-rec_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_open-rec_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/open-rec/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_research_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_research_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/research/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_newsroom_events_calendar.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 208 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_newsroom_events_calendar.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 208 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/newsroom/events/calendar.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_external_s-dmv.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 173 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_external_s-dmv.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 173 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/external/s-dmv.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_DivEqInc.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 233 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_DivEqInc.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 233 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/DivEqInc.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_newsroom_Logo-photo-gal_DefautlLogographicgal.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 227 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_newsroom_Logo-photo-gal_DefautlLogographicgal.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 227 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/newsroom/Logo-photo-gal/DefautlLogographicgal.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_newsroom_statistics_general.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 151 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_newsroom_statistics_general.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 151 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/newsroom/statistics/general.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_newsroom_law_lom.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 243 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_newsroom_law_lom.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 243 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/newsroom/law/lom.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_newsroom_media_contacts.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 216 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_newsroom_media_contacts.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 216 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/newsroom/media/contacts.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_newsroom_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 206 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_newsroom_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 206 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/newsroom/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_careers_equal.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 230 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_careers_equal.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 230 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/careers/equal.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_performance_open-book_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_performance_open-book_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/performance/open-book/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_careers_benefits.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 210 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_careers_benefits.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 210 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/careers/benefits.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_careers_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 216 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_careers_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 216 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/careers/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_contact-us_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 216 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_contact-us_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 216 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/contact-us/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_performance_budget_budget.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 208 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_performance_budget_budget.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 208 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/performance/budget/budget.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_performance_continuous-improvement.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 234 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_performance_continuous-improvement.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 234 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/performance/continuous-improvement.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_performance_fed-measures_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 245 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_performance_fed-measures_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 245 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/performance/fed-measures/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dtsd_dtsd-region-offices.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 269 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dtsd_dtsd-region-offices.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 269 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtsd/dtsd-region-offices.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dsp_loc-contact.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 166 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dsp_loc-contact.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 166 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dsp/loc-contact.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_contact-us_hill-farms-bldg.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 234 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_contact-us_hill-farms-bldg.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 234 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/contact-us/hill-farms-bldg.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_comm-couns_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 226 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_comm-couns_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 226 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/comm-couns/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dtsd_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 247 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dtsd_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 247 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtsd/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dtim_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 250 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dtim_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 250 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtim/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dept-overview_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dept-overview_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dept-overview/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_exec-offc_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_exec-offc_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/exec-offc/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dbm_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 233 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dbm_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 233 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dbm/default.aspx\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-26 08:19:57 [scrapy.extensions.logstats] INFO: Crawled 111 pages (at 111 pages/min), scraped 104 items (at 104 items/min)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: [query/web_datasource.py:46 - save_content()] Saving .html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 311 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading .html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 311 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://mapss.wisconsindot.gov/\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dsp_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 233 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dsp_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 233 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dsp/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dmv_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 234 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dmv_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 234 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dmv/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_about-wisdot_who-we-are_dbsi.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 247 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_about-wisdot_who-we-are_dbsi.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 247 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dbsi.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_solvency.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 235 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_solvency.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 235 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/solvency.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_pedplan2020.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 239 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_pedplan2020.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 239 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/pedplan2020.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_6yr-hwy-impr_proj-info_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 248 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_6yr-hwy-impr_proj-info_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 248 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/proj-info/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_bikeplan2020.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 244 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_bikeplan2020.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 244 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/bikeplan2020.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_sasp_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 242 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_sasp_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 242 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/sasp/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_6yr-hwy-impr_state-hwy_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 247 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_6yr-hwy-impr_state-hwy_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 247 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/state-hwy/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_6yr-hwy-impr_maj-hwy_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 238 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_6yr-hwy-impr_maj-hwy_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 238 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/maj-hwy/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_6yr-hwy-impr_overview_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 249 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_6yr-hwy-impr_overview_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 249 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/overview/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_lif-hwy-proj_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 227 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_lif-hwy-proj_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 227 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/lif-hwy-proj/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_eng-consultants_cnslt-rsrces_environment_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 236 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_eng-consultants_cnslt-rsrces_environment_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 236 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rsrces/environment/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_data-plan_traf-fore_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_data-plan_traf-fore_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/data-plan/traf-fore/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_data-plan_traf-counts_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 216 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_data-plan_traf-counts_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 216 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/data-plan/traf-counts/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_data-plan_aerial-image_aerial.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 216 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_data-plan_aerial-image_aerial.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 216 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/data-plan/aerial-image/aerial.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_astnce-pgms_highway_stip.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 253 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_astnce-pgms_highway_stip.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 253 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/stip.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_education_crash-data_crashfacts.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 228 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_education_crash-data_crashfacts.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 228 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/education/crash-data/crashfacts.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_data-plan_veh-miles_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 231 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_data-plan_veh-miles_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 231 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/data-plan/veh-miles/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_data-plan_plan-res_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 220 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_data-plan_plan-res_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 220 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/data-plan/plan-res/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_water.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 228 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_water.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 228 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/water.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_transit.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 230 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_transit.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 230 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/transit.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_rail.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 231 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_rail.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 231 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/rail.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_cav.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 128 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_cav.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 128 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/cav.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_bike.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 230 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_bike.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 230 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/bike.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_air.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 226 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_air.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 226 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/air.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_by-region_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 127 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_by-region_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 127 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/by-region/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_enforcement_agencies_grants.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 243 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_enforcement_agencies_grants.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 243 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/enforcement/agencies/grants.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_multimodal_ped.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 233 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_multimodal_ped.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 233 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/multimodal/ped.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_veh-inspect_salvage-veh_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 226 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_veh-inspect_salvage-veh_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 226 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/veh-inspect/salvage-veh/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_veh-inspect_human-srvc_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 224 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_veh-inspect_human-srvc_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 224 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/veh-inspect/human-srvc/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_veh-inspect_reconst-veh_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 236 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_veh-inspect_reconst-veh_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 236 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/veh-inspect/reconst-veh/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_astnce-pgms_highway_hsip.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 243 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_astnce-pgms_highway_hsip.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 243 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/hsip.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_damage-claim_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 226 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_damage-claim_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 226 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/damage-claim/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_veh-inspect_faqs.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 241 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_veh-inspect_faqs.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 241 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/veh-inspect/faqs.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_veh-inspect_sch-bus_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 223 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_veh-inspect_sch-bus_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 223 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/veh-inspect/sch-bus/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_veh-inspect_farm-veh_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 215 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_veh-inspect_farm-veh_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 215 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/veh-inspect/farm-veh/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_veh-inspect_motor-bus_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_veh-inspect_motor-bus_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/veh-inspect/motor-bus/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_veh-inspect_ambulance_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_veh-inspect_ambulance_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/veh-inspect/ambulance/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_humantrfkng.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_humantrfkng.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/humantrfkng.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_equipment_sch-bus_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_equipment_sch-bus_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/equipment/sch-bus/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_motorcycles_mc-safety_gear.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 220 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_motorcycles_mc-safety_gear.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 220 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/motorcycles/mc-safety/gear.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_education_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 226 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_education_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 226 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/education/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_enforcement_faqs_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 242 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_enforcement_faqs_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 242 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/enforcement/faqs/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_enforcement_citation_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 250 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_enforcement_citation_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 250 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/enforcement/citation/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_crsh-rpt_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 217 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_crsh-rpt_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 217 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/crsh-rpt/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_safety-eng_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 220 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_safety-eng_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 220 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/safety-eng/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_mtr-car-trkr_mc-safety_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 227 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_mtr-car-trkr_mc-safety_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 227 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/mc-safety/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_air_airport-info_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_air_airport-info_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/air/airport-info/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_crsh-rpt_iroc.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_crsh-rpt_iroc.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/crsh-rpt/iroc.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_safety_enforcement_agencies_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 217 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_safety_enforcement_agencies_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 217 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/safety/enforcement/agencies/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_pub-transit_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 226 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_pub-transit_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 226 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/pub-transit/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_air_pilot-info_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_air_pilot-info_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/air/pilot-info/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_water_cruise.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 231 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_water_cruise.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 231 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/water/cruise.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_water_ferries.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 232 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_water_ferries.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 232 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/water/ferries.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_rail_mis.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 230 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_rail_mis.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 230 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/rail/mis.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_ped_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 224 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_ped_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 224 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/ped/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_bike_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 216 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_bike_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 216 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/bike/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_rail_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 216 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_rail_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 216 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/rail/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_rail_mars.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 236 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_rail_mars.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 236 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/rail/mars.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_rail_amtrak.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 231 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_rail_amtrak.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 231 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/rail/amtrak.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_rustic-roads_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 214 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_rustic-roads_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 214 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/rustic-roads/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_out-of-state_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_out-of-state_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/out-of-state/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_scenic-ways_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 233 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_scenic-ways_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 233 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/scenic-ways/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_rideshare_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 125 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_rideshare_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 125 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/rideshare/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_rest-areas_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_rest-areas_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/rest-areas/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_parknride_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 121 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_parknride_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 121 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/parknride/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_flexlane_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 118 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_flexlane_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 118 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/flexlane/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_comm-hwys_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 236 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_comm-hwys_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 236 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/comm-hwys/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_road_hwy-maps_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 232 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_road_hwy-maps_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 232 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/road/hwy-maps/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_travel_511_511.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_travel_511_511.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/travel/511/511.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_MapsGIS.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 214 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_MapsGIS.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 214 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/MapsGIS.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_projects_in-together_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 218 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_projects_in-together_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 218 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/projects/in-together/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_freight_contacts.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 225 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_freight_contacts.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 225 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/freight/contacts.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_freight_resources.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_freight_resources.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/freight/resources.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_freight_econ-dev.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_freight_econ-dev.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/freight/econ-dev.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_freight_rail.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_freight_rail.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/freight/rail.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_real-estate_outdoor-adv_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 227 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_real-estate_outdoor-adv_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 227 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/real-estate/outdoor-adv/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_freight_fac.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 228 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_freight_fac.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 228 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/freight/fac.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_real-estate_roadsides_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 211 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_real-estate_roadsides_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 211 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/real-estate/roadsides/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_real-estate_access-mgmt_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_real-estate_access-mgmt_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/real-estate/access-mgmt/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_real-estate_permits_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 234 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_real-estate_permits_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 234 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/real-estate/permits/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_freight_summit.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 236 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_freight_summit.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 236 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/freight/summit.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_real-estate_landsales_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 245 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_real-estate_landsales_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 245 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/real-estate/landsales/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_real-estate_hghwys-yourland_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 236 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_real-estate_hghwys-yourland_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 236 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/real-estate/hghwys-yourland/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_aeronautics_trng-evnts_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 230 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_aeronautics_trng-evnts_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 230 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/aeronautics/trng-evnts/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_aeronautics_education_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 220 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_aeronautics_education_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 220 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/aeronautics/education/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_aeronautics_resources_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 219 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_aeronautics_resources_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 219 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/aeronautics/resources/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_aeronautics_ac-rgstrtn_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 223 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_aeronautics_ac-rgstrtn_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 223 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/aeronautics/ac-rgstrtn/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_aeronautics_airports_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 239 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_aeronautics_airports_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 239 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/aeronautics/airports/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_aeronautics_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_aeronautics_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/aeronautics/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_wislr_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 254 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_wislr_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 254 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/wislr/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_plning-orgs_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 224 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_plning-orgs_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 224 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/plning-orgs/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_hwy-mnt_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_hwy-mnt_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/hwy-mnt/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_astnce-pgms_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 227 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_astnce-pgms_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 227 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_vehicles_prkg-tckt_unpaid-tickets.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 198 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_vehicles_prkg-tckt_unpaid-tickets.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 198 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/vehicles/prkg-tckt/unpaid-tickets.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_astnce-pgms_highway_TAS.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 240 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_astnce-pgms_highway_TAS.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 240 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/TAS.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_traffic-ops_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 220 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_traffic-ops_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 220 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/traffic-ops/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_project-invoices.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 218 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_project-invoices.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 218 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/project-invoices.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_local-gov_lpm_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 247 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_local-gov_lpm_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 247 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/local-gov/lpm/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_purchasing_suply-dvrsty_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 235 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_purchasing_suply-dvrsty_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 235 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/purchasing/suply-dvrsty/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_purchasing_invoice-nqry_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 218 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_purchasing_invoice-nqry_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 218 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/purchasing/invoice-nqry/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_purchasing_who-buy-what_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 225 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_purchasing_who-buy-what_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 225 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/purchasing/who-buy-what/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_purchasing_vendornet_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 233 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_purchasing_vendornet_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 233 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/purchasing/vendornet/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_civil-rights_dbe_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 249 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_civil-rights_dbe_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 249 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/civil-rights/dbe/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_purchasing_prchsg-ovrvw_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_purchasing_prchsg-ovrvw_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/purchasing/prchsg-ovrvw/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_civil-rights_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 229 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_civil-rights_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 229 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/civil-rights/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_designbuild_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_designbuild_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/designbuild/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_contractors_cntrctr-trng_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 210 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_contractors_cntrctr-trng_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 210 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/contractors/cntrctr-trng/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_contractors_hcci_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 243 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_contractors_hcci_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 243 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/contractors/hcci/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_eng-consultants_cnslt-cntct_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 221 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_eng-consultants_cnslt-cntct_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 221 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-cntct/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_eng-consultants_partner_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 223 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_eng-consultants_partner_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 223 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/eng-consultants/partner/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_contractors_tech-teams_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 222 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_contractors_tech-teams_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 222 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/contractors/tech-teams/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_contractors_cntrct-logs_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 236 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_contractors_cntrct-logs_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 236 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/contractors/cntrct-logs/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_contractors_cntrct-pymts_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 247 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_contractors_cntrct-pymts_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 247 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/contractors/cntrct-pymts/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_eng-consultants_cnslt-rsrces_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 230 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_eng-consultants_cnslt-rsrces_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 230 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rsrces/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_eng-consultants_cnslt-rgistr_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 210 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_eng-consultants_cnslt-rgistr_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 210 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rgistr/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_eng-consultants_selections_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 212 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_eng-consultants_selections_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 212 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/eng-consultants/selections/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_eng-consultants_solicitations_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 225 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_eng-consultants_solicitations_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 225 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/eng-consultants/solicitations/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_eng-consultants_notices_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 220 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_eng-consultants_notices_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 220 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/eng-consultants/notices/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_doing-bus_bil.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 245 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_doing-bus_bil.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 245 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/doing-bus/bil.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_agri-eq-veh_faqs.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 228 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_agri-eq-veh_faqs.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 228 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/agri-eq-veh/faqs.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_agri-eq-veh_ag-veh-safety.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 229 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_agri-eq-veh_ag-veh-safety.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 229 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-veh-safety.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_agri-eq-veh_ag-prmts.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 228 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_agri-eq-veh_ag-prmts.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 228 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-prmts.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_agri-eq-veh_local-govt-info.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 230 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_agri-eq-veh_local-govt-info.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 230 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/agri-eq-veh/local-govt-info.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_mtr-car-trkr_default.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 164 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_mtr-car-trkr_default.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 164 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/default.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_cdl-med-rqts_cdl-medical.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 169 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_cdl-med-rqts_cdl-medical.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 169 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-med-rqts/cdl-medical.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_cdl-how-aply_cdlapply.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 154 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_cdl-how-aply_cdlapply.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 154 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-how-aply/cdlapply.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_rqts-for-cdl_cdl-requirements.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 165 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_rqts-for-cdl_cdl-requirements.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 165 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/cdl-requirements.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_external_bvs-landing.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 167 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_external_bvs-landing.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 167 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/external/bvs-landing.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_external_bds-landing.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 156 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_external_bds-landing.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 156 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/external/bds-landing.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_dmv_com-drv-vehs_rqts-for-cdl_schoolbusinfo.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 255 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_dmv_com-drv-vehs_rqts-for-cdl_schoolbusinfo.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 255 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/schoolbusinfo.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_VehicleServices.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 226 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_VehicleServices.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 226 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/VehicleServices.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_DLID-Services.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 235 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_DLID-Services.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 235 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/DLID-Services.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_other-servs_Howto.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 228 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_other-servs_Howto.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 228 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/other-servs/Howto.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_online-srvcs_online.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 152 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_online-srvcs_online.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 152 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/online-srvcs/online.aspx\n",
      "INFO: [query/web_datasource.py:46 - save_content()] Saving Pages_Home.aspx.html to /var/folders/65/hxytdjv5109ct635w2lnnj6w00dhhl/T/tmp5gpskqj1\n",
      "INFO: [query/web_datasource.py:50 - save_content()] 134 bytes written\n",
      "INFO: [utils/gcs_helper.py:55 - upload_to_gcs()] Uploading Pages_Home.aspx.html to GCS bucket lramsey-dev-downloads-test-vertex-search-6\n",
      "INFO: [utils/gcs_helper.py:62 - upload_to_gcs()] Uploaded 134 bytes\n",
      "INFO: [query/web_datasource.py:209 - _item_scraped()] Downloaded Response URL: https://wisconsindot.gov/Pages/Home.aspx\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-26 08:20:50 [scrapy.core.engine] INFO: Closing spider (finished)\n",
      "2024-02-26 08:20:50 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 113862,\n",
      " 'downloader/request_count': 244,\n",
      " 'downloader/request_method_count/GET': 244,\n",
      " 'downloader/response_bytes': 37360370,\n",
      " 'downloader/response_count': 244,\n",
      " 'downloader/response_status_count/200': 244,\n",
      " 'elapsed_time_seconds': 113.732351,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2024, 2, 26, 13, 20, 50, 583498, tzinfo=datetime.timezone.utc),\n",
      " 'item_scraped_count': 243,\n",
      " 'log_count/INFO': 11,\n",
      " 'log_count/WARNING': 1,\n",
      " 'memusage/max': 506593280,\n",
      " 'memusage/startup': 418586624,\n",
      " 'request_depth_max': 1,\n",
      " 'response_received_count': 244,\n",
      " 'scheduler/dequeued': 244,\n",
      " 'scheduler/dequeued/memory': 244,\n",
      " 'scheduler/enqueued': 244,\n",
      " 'scheduler/enqueued/memory': 244,\n",
      " 'start_time': datetime.datetime(2024, 2, 26, 13, 18, 56, 851147, tzinfo=datetime.timezone.utc)}\n",
      "2024-02-26 08:20:50 [scrapy.core.engine] INFO: Spider closed (finished)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished\n",
      "https://wisconsindot.gov/pages/online-srvcs/external/dmv.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/cons-protect/buy-sell-veh/buyorsellavehicle.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/cons-protect/lemon-law/lemonlaw.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ioh.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-cmv.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/cons-protect/know-rghts/knowyourrights.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/agri-eq-veh/axle-weights.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/cons-protect/file-cmplnt/filedealercomplaint.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/agri-eq-veh/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-training/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-inds-plt/dlrspecialplate.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/dlr-agents/tips-tls-dlr/tips.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-frms-pbs/dealer-forms.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-ttl-reg/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/dlr-agents/busns-lcnse/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-training/dealer.aspx\n",
      "https://wisconsindot.gov/Pages/global-footer/translate.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/careers/employees.aspx\n",
      "https://wisconsindot.gov/Pages/global-footer/software.aspx\n",
      "https://wisconsindot.gov/Pages/global-footer/Privacy-notice.aspx\n",
      "https://wisconsindot.gov/Pages/global-footer/Legal-notices.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/accessibility-statement.aspx\n",
      "https://wisconsindot.gov/Pages/global-footer/Acceptable-use-policy.aspx\n",
      "https://wisconsindot.gov/Pages/global-footer/Support.aspx\n",
      "https://wisconsindot.gov/Pages/global-footer/formdocs/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/dmvagent.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/contact-us/dmv-contact.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/frms-pubs/veh-forms.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/incidents.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/prkg-tckt/Web-TVRP.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/find-dmv/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/rcd-crsh-rpt/optout.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/rcd-crsh-rpt/driver-forms.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/req-veh-record.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/temp-discard.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/request-record.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/sll-jnk-vhcl/sellyourvehicle.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/discards.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/displate.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/special-list.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/rnew-plts/emissiontest.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/nolaf.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/dup-c-of-r.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/new-res-default.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/renew-licens/plate-default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/osow.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/audit.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/irp.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/self-cert.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/mc-ins.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-how-aply/practiceapplication.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/fmcsa-reg-update-info.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/ifta.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/walkin.aspx\n",
      "https://wisconsindot.gov/pages/dmv/license-drvs/rcd-crsh-rpt/auto-insurance.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/susp-or-rvkd/owi.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/handbook.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/donor.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/veterans.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/mdcl-cncrns/med-concerns.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/mdcl-cncrns/olderdrivers.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/id-card.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/license-renewal.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/motorcycles/mc-how-aply/schedulearoadtest.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/status.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/duplicate-id.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/petition-process.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/status-check.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/lst-or-stoln/duplicate-id.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/susp-or-rvkd/reinstate.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/name-change.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/change-addy/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/get-lic.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/teen-driver/teen-sfty/index.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/roadtestgeneral.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/duplicate-license.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/title-vehicle/emvpublicdefault.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/open-rec/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/research/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/newsroom/events/calendar.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/external/s-dmv.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/DivEqInc.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/newsroom/Logo-photo-gal/DefautlLogographicgal.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/newsroom/statistics/general.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/newsroom/law/lom.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/newsroom/media/contacts.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/newsroom/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/careers/equal.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/performance/open-book/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/careers/benefits.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/careers/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/contact-us/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/performance/budget/budget.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/performance/continuous-improvement.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/performance/fed-measures/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtsd/dtsd-region-offices.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dsp/loc-contact.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/contact-us/hill-farms-bldg.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/comm-couns/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtsd/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtim/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dept-overview/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/exec-offc/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dbm/default.aspx\n",
      "https://mapss.wisconsindot.gov/\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dsp/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dmv/default.aspx\n",
      "https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dbsi.aspx\n",
      "https://wisconsindot.gov/Pages/projects/solvency.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/pedplan2020.aspx\n",
      "https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/proj-info/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/bikeplan2020.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/sasp/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/state-hwy/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/maj-hwy/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/overview/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/lif-hwy-proj/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rsrces/environment/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/data-plan/traf-fore/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/data-plan/traf-counts/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/data-plan/aerial-image/aerial.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/stip.aspx\n",
      "https://wisconsindot.gov/Pages/safety/education/crash-data/crashfacts.aspx\n",
      "https://wisconsindot.gov/Pages/projects/data-plan/veh-miles/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/data-plan/plan-res/default.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/water.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/transit.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/rail.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/cav.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/bike.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/air.aspx\n",
      "https://wisconsindot.gov/Pages/projects/by-region/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/enforcement/agencies/grants.aspx\n",
      "https://wisconsindot.gov/Pages/projects/multimodal/ped.aspx\n",
      "https://wisconsindot.gov/Pages/safety/veh-inspect/salvage-veh/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/veh-inspect/human-srvc/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/veh-inspect/reconst-veh/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/hsip.aspx\n",
      "https://wisconsindot.gov/Pages/safety/damage-claim/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/veh-inspect/faqs.aspx\n",
      "https://wisconsindot.gov/Pages/safety/veh-inspect/sch-bus/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/veh-inspect/farm-veh/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/veh-inspect/motor-bus/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/veh-inspect/ambulance/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/humantrfkng.aspx\n",
      "https://wisconsindot.gov/Pages/safety/equipment/sch-bus/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/motorcycles/mc-safety/gear.aspx\n",
      "https://wisconsindot.gov/Pages/safety/education/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/enforcement/faqs/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/enforcement/citation/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/crsh-rpt/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/safety-eng/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/mc-safety/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/air/airport-info/default.aspx\n",
      "https://wisconsindot.gov/Pages/safety/crsh-rpt/iroc.aspx\n",
      "https://wisconsindot.gov/Pages/safety/enforcement/agencies/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/pub-transit/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/air/pilot-info/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/water/cruise.aspx\n",
      "https://wisconsindot.gov/Pages/travel/water/ferries.aspx\n",
      "https://wisconsindot.gov/Pages/travel/rail/mis.aspx\n",
      "https://wisconsindot.gov/Pages/travel/ped/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/bike/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/rail/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/rail/mars.aspx\n",
      "https://wisconsindot.gov/Pages/travel/rail/amtrak.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/rustic-roads/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/out-of-state/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/scenic-ways/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/rideshare/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/rest-areas/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/parknride/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/flexlane/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/comm-hwys/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/road/hwy-maps/default.aspx\n",
      "https://wisconsindot.gov/Pages/travel/511/511.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/MapsGIS.aspx\n",
      "https://wisconsindot.gov/Pages/projects/in-together/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/freight/contacts.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/freight/resources.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/freight/econ-dev.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/freight/rail.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/real-estate/outdoor-adv/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/freight/fac.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/real-estate/roadsides/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/real-estate/access-mgmt/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/real-estate/permits/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/freight/summit.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/real-estate/landsales/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/real-estate/hghwys-yourland/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/aeronautics/trng-evnts/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/aeronautics/education/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/aeronautics/resources/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/aeronautics/ac-rgstrtn/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/aeronautics/airports/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/aeronautics/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/wislr/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/plning-orgs/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/hwy-mnt/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/vehicles/prkg-tckt/unpaid-tickets.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/TAS.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/traffic-ops/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/project-invoices.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/local-gov/lpm/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/purchasing/suply-dvrsty/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/purchasing/invoice-nqry/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/purchasing/who-buy-what/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/purchasing/vendornet/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/civil-rights/dbe/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/purchasing/prchsg-ovrvw/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/civil-rights/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/designbuild/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/contractors/cntrctr-trng/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/contractors/hcci/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-cntct/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/eng-consultants/partner/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/contractors/tech-teams/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/contractors/cntrct-logs/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/contractors/cntrct-pymts/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rsrces/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rgistr/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/eng-consultants/selections/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/eng-consultants/solicitations/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/eng-consultants/notices/default.aspx\n",
      "https://wisconsindot.gov/Pages/doing-bus/bil.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/agri-eq-veh/faqs.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-veh-safety.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-prmts.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/agri-eq-veh/local-govt-info.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/default.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-med-rqts/cdl-medical.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-how-aply/cdlapply.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/cdl-requirements.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/external/bvs-landing.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/external/bds-landing.aspx\n",
      "https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/schoolbusinfo.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/VehicleServices.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/DLID-Services.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/other-servs/Howto.aspx\n",
      "https://wisconsindot.gov/Pages/online-srvcs/online.aspx\n",
      "https://wisconsindot.gov/Pages/Home.aspx\n",
      "INFO: [query/web_datasource.py:268 - download_documents()] Scraped 243 links\n"
     ]
    }
   ],
   "source": [
    "gcs_url, web_docs_downloaded = download_web_docs(q_engine, data_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "98d35ca3-a9fc-47bd-8f14-4c38bfe34e12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://wisconsindot.gov/pages/online-srvcs/external/dmv.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/cons-protect/buy-sell-veh/buyorsellavehicle.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/cons-protect/lemon-law/lemonlaw.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ioh.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-cmv.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/cons-protect/know-rghts/knowyourrights.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/agri-eq-veh/axle-weights.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/cons-protect/file-cmplnt/filedealercomplaint.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/agri-eq-veh/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-training/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-inds-plt/dlrspecialplate.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/dlr-agents/tips-tls-dlr/tips.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-frms-pbs/dealer-forms.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-ttl-reg/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/dlr-agents/busns-lcnse/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/dlr-agents/dlr-training/dealer.aspx',\n",
       " 'https://wisconsindot.gov/Pages/global-footer/translate.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/careers/employees.aspx',\n",
       " 'https://wisconsindot.gov/Pages/global-footer/software.aspx',\n",
       " 'https://wisconsindot.gov/Pages/global-footer/Privacy-notice.aspx',\n",
       " 'https://wisconsindot.gov/Pages/global-footer/Legal-notices.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/accessibility-statement.aspx',\n",
       " 'https://wisconsindot.gov/Pages/global-footer/Acceptable-use-policy.aspx',\n",
       " 'https://wisconsindot.gov/Pages/global-footer/Support.aspx',\n",
       " 'https://wisconsindot.gov/Pages/global-footer/formdocs/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/dmvagent.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/contact-us/dmv-contact.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/frms-pubs/veh-forms.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/incidents.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/prkg-tckt/Web-TVRP.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/find-dmv/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/rcd-crsh-rpt/optout.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/rcd-crsh-rpt/driver-forms.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/req-veh-record.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/temp-discard.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/request-record.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/sll-jnk-vhcl/sellyourvehicle.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/discards.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/dsbld-prkg/displate.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/special-list.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/rnew-plts/emissiontest.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/nolaf.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/dup-c-of-r.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/new-res-default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/renew-licens/plate-default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/osow.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/audit.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/irp.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/self-cert.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/mc-ins.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-how-aply/practiceapplication.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/fmcsa-reg-update-info.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/ifta.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/walkin.aspx',\n",
       " 'https://wisconsindot.gov/pages/dmv/license-drvs/rcd-crsh-rpt/auto-insurance.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/susp-or-rvkd/owi.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/handbook.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/donor.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/veterans.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/mdcl-cncrns/med-concerns.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/mdcl-cncrns/olderdrivers.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/id-card.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/license-renewal.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/motorcycles/mc-how-aply/schedulearoadtest.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/title-plates/status.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/duplicate-id.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/petition-process.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/status-check.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/lst-or-stoln/duplicate-id.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/susp-or-rvkd/reinstate.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/rnew-and-chge/name-change.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/change-addy/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/get-lic.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/teen-driver/teen-sfty/index.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/license-drvs/how-to-apply/roadtestgeneral.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/duplicate-license.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/title-vehicle/emvpublicdefault.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/open-rec/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/research/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/newsroom/events/calendar.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/external/s-dmv.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/DivEqInc.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/newsroom/Logo-photo-gal/DefautlLogographicgal.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/newsroom/statistics/general.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/newsroom/law/lom.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/newsroom/media/contacts.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/newsroom/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/careers/equal.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/performance/open-book/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/careers/benefits.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/careers/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/contact-us/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/performance/budget/budget.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/performance/continuous-improvement.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/performance/fed-measures/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtsd/dtsd-region-offices.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dsp/loc-contact.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/contact-us/hill-farms-bldg.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/comm-couns/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtsd/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dtim/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dept-overview/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/exec-offc/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dbm/default.aspx',\n",
       " 'https://mapss.wisconsindot.gov/',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dsp/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dmv/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/about-wisdot/who-we-are/dbsi.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/solvency.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/pedplan2020.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/proj-info/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/bikeplan2020.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/sasp/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/state-hwy/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/maj-hwy/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/6yr-hwy-impr/overview/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/lif-hwy-proj/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rsrces/environment/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/data-plan/traf-fore/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/data-plan/traf-counts/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/data-plan/aerial-image/aerial.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/stip.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/education/crash-data/crashfacts.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/data-plan/veh-miles/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/data-plan/plan-res/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/water.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/transit.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/rail.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/cav.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/bike.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/air.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/by-region/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/enforcement/agencies/grants.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/multimodal/ped.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/veh-inspect/salvage-veh/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/veh-inspect/human-srvc/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/veh-inspect/reconst-veh/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/hsip.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/damage-claim/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/veh-inspect/faqs.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/veh-inspect/sch-bus/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/veh-inspect/farm-veh/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/veh-inspect/motor-bus/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/veh-inspect/ambulance/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/humantrfkng.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/equipment/sch-bus/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/motorcycles/mc-safety/gear.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/education/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/enforcement/faqs/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/enforcement/citation/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/crsh-rpt/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/safety-eng/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/mc-safety/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/air/airport-info/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/crsh-rpt/iroc.aspx',\n",
       " 'https://wisconsindot.gov/Pages/safety/enforcement/agencies/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/pub-transit/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/air/pilot-info/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/water/cruise.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/water/ferries.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/rail/mis.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/ped/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/bike/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/rail/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/rail/mars.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/rail/amtrak.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/rustic-roads/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/out-of-state/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/scenic-ways/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/rideshare/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/rest-areas/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/parknride/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/flexlane/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/comm-hwys/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/road/hwy-maps/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/travel/511/511.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/MapsGIS.aspx',\n",
       " 'https://wisconsindot.gov/Pages/projects/in-together/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/freight/contacts.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/freight/resources.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/freight/econ-dev.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/freight/rail.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/real-estate/outdoor-adv/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/freight/fac.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/real-estate/roadsides/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/real-estate/access-mgmt/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/real-estate/permits/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/freight/summit.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/real-estate/landsales/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/real-estate/hghwys-yourland/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/aeronautics/trng-evnts/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/aeronautics/education/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/aeronautics/resources/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/aeronautics/ac-rgstrtn/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/aeronautics/airports/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/aeronautics/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/wislr/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/plning-orgs/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/hwy-mnt/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/vehicles/prkg-tckt/unpaid-tickets.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/astnce-pgms/highway/TAS.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/traffic-ops/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/project-invoices.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/local-gov/lpm/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/purchasing/suply-dvrsty/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/purchasing/invoice-nqry/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/purchasing/who-buy-what/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/purchasing/vendornet/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/civil-rights/dbe/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/purchasing/prchsg-ovrvw/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/civil-rights/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/designbuild/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/contractors/cntrctr-trng/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/contractors/hcci/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-cntct/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/eng-consultants/partner/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/contractors/tech-teams/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/contractors/cntrct-logs/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/contractors/cntrct-pymts/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rsrces/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/eng-consultants/cnslt-rgistr/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/eng-consultants/selections/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/eng-consultants/solicitations/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/eng-consultants/notices/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/doing-bus/bil.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/agri-eq-veh/faqs.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-veh-safety.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/agri-eq-veh/ag-prmts.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/agri-eq-veh/local-govt-info.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/mtr-car-trkr/default.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-med-rqts/cdl-medical.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/cdl-how-aply/cdlapply.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/cdl-requirements.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/external/bvs-landing.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/external/bds-landing.aspx',\n",
       " 'https://wisconsindot.gov/Pages/dmv/com-drv-vehs/rqts-for-cdl/schoolbusinfo.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/VehicleServices.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/DLID-Services.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/other-servs/Howto.aspx',\n",
       " 'https://wisconsindot.gov/Pages/online-srvcs/online.aspx',\n",
       " 'https://wisconsindot.gov/Pages/Home.aspx']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "web_docs_downloaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8a9f05fa-84c2-47a5-8fc2-407f43f6fa5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gs://lramsey-dev-downloads-test-vertex-search-6'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gcs_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "45089237-7801-4807-82e0-63d1cdbe5c95",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_to_be_processed = inventory_gcs_files(gcs_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "dfc68acc-5a8b-42e0-9cd0-df24e9148c79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gs://lramsey-dev-downloads-test-vertex-search-6/Pages_Home.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_DivEqInc.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_accessibility-statement.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_careers_benefits.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_careers_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_careers_employees.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_careers_equal.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_contact-us_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_contact-us_dmv-contact.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_contact-us_hill-farms-bldg.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_newsroom_Logo-photo-gal_DefautlLogographicgal.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_newsroom_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_newsroom_events_calendar.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_newsroom_law_lom.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_newsroom_media_contacts.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_newsroom_statistics_general.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_open-rec_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_performance_budget_budget.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_performance_continuous-improvement.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_performance_fed-measures_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_performance_open-book_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_research_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_comm-couns_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dbm_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dbsi.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dept-overview_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dmv_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dsp_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dsp_loc-contact.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dtim_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dtsd_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_dtsd_dtsd-region-offices.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_about-wisdot_who-we-are_exec-offc_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_agri-eq-veh_ag-cmv.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_agri-eq-veh_ag-prmts.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_agri-eq-veh_ag-veh-safety.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_agri-eq-veh_axle-weights.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_agri-eq-veh_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_agri-eq-veh_faqs.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_agri-eq-veh_ioh.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_agri-eq-veh_local-govt-info.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_cdl-how-aply_cdlapply.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_cdl-how-aply_practiceapplication.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_cdl-med-rqts_cdl-medical.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_mtr-car-trkr_audit.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_mtr-car-trkr_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_mtr-car-trkr_ifta.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_mtr-car-trkr_irp.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_mtr-car-trkr_mc-ins.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_mtr-car-trkr_mc-safety_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_rqts-for-cdl_cdl-requirements.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_rqts-for-cdl_fmcsa-reg-update-info.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_com-drv-vehs_rqts-for-cdl_schoolbusinfo.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_cons-protect_buy-sell-veh_buyorsellavehicle.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_cons-protect_file-cmplnt_filedealercomplaint.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_cons-protect_know-rghts_knowyourrights.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_cons-protect_lemon-law_lemonlaw.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_dlr-agents_busns-lcnse_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_dlr-agents_dlr-frms-pbs_dealer-forms.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_dlr-agents_dlr-inds-plt_dlrspecialplate.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_dlr-agents_dlr-training_dealer.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_dlr-agents_dlr-training_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_dlr-agents_dlr-ttl-reg_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_dlr-agents_tips-tls-dlr_tips.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_how-to-apply_get-lic.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_how-to-apply_handbook.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_how-to-apply_id-card.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_how-to-apply_petition-process.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_how-to-apply_roadtestgeneral.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_lst-or-stoln_duplicate-id.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_mdcl-cncrns_med-concerns.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_mdcl-cncrns_olderdrivers.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_rcd-crsh-rpt_driver-forms.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_rcd-crsh-rpt_optout.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_rnew-and-chge_donor.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_rnew-and-chge_license-renewal.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_rnew-and-chge_name-change.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_rnew-and-chge_veterans.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_susp-or-rvkd_owi.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_license-drvs_susp-or-rvkd_reinstate.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_motorcycles_mc-how-aply_schedulearoadtest.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_motorcycles_mc-safety_gear.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_teen-driver_teen-sfty_index.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_dsbld-prkg_discards.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_dsbld-prkg_displate.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_dsbld-prkg_temp-discard.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_frms-pubs_veh-forms.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_prkg-tckt_Web-TVRP.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_prkg-tckt_unpaid-tickets.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_rnew-plts_emissiontest.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_sll-jnk-vhcl_sellyourvehicle.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_title-plates_dmvagent.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_title-plates_new-res-default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_title-plates_nolaf.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_title-plates_osow.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_title-plates_special-list.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_title-plates_status.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_dmv_vehicles_title-plates_walkin.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_MapsGIS.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_aeronautics_ac-rgstrtn_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_aeronautics_airports_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_aeronautics_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_aeronautics_education_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_aeronautics_resources_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_aeronautics_trng-evnts_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_bil.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_civil-rights_dbe_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_civil-rights_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_contractors_cntrct-logs_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_contractors_cntrct-pymts_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_contractors_cntrctr-trng_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_contractors_hcci_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_contractors_tech-teams_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_designbuild_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_eng-consultants_cnslt-cntct_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_eng-consultants_cnslt-rgistr_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_eng-consultants_cnslt-rsrces_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_eng-consultants_cnslt-rsrces_environment_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_eng-consultants_notices_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_eng-consultants_partner_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_eng-consultants_selections_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_eng-consultants_solicitations_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_freight_contacts.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_freight_econ-dev.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_freight_fac.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_freight_rail.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_freight_resources.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_freight_summit.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_astnce-pgms_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_astnce-pgms_highway_TAS.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_astnce-pgms_highway_hsip.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_astnce-pgms_highway_stip.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_hwy-mnt_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_lpm_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_plning-orgs_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_project-invoices.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_traffic-ops_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_local-gov_wislr_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_purchasing_invoice-nqry_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_purchasing_prchsg-ovrvw_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_purchasing_suply-dvrsty_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_purchasing_vendornet_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_purchasing_who-buy-what_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_real-estate_access-mgmt_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_real-estate_hghwys-yourland_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_real-estate_landsales_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_real-estate_outdoor-adv_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_real-estate_permits_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_doing-bus_real-estate_roadsides_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_global-footer_Acceptable-use-policy.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_global-footer_Legal-notices.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_global-footer_Privacy-notice.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_global-footer_Support.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_global-footer_formdocs_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_global-footer_software.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_global-footer_translate.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_DLID-Services.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_VehicleServices.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_change-addy_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_external_bds-landing.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_external_bvs-landing.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_external_s-dmv.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_find-dmv_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_online.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_Howto.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_dup-c-of-r.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_duplicate-id.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_duplicate-license.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_incidents.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_req-veh-record.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_request-record.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_self-cert.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_other-servs_status-check.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_renew-licens_plate-default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_online-srvcs_title-vehicle_emvpublicdefault.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_6yr-hwy-impr_maj-hwy_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_6yr-hwy-impr_overview_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_6yr-hwy-impr_proj-info_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_6yr-hwy-impr_state-hwy_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_by-region_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_data-plan_aerial-image_aerial.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_data-plan_plan-res_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_data-plan_traf-counts_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_data-plan_traf-fore_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_data-plan_veh-miles_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_in-together_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_lif-hwy-proj_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_air.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_bike.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_bikeplan2020.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_cav.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_ped.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_pedplan2020.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_rail.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_sasp_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_transit.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_multimodal_water.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_projects_solvency.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_crsh-rpt_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_crsh-rpt_iroc.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_damage-claim_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_education_crash-data_crashfacts.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_education_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_enforcement_agencies_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_enforcement_agencies_grants.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_enforcement_citation_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_enforcement_faqs_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_equipment_sch-bus_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_humantrfkng.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_safety-eng_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_veh-inspect_ambulance_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_veh-inspect_faqs.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_veh-inspect_farm-veh_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_veh-inspect_human-srvc_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_veh-inspect_motor-bus_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_veh-inspect_reconst-veh_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_veh-inspect_salvage-veh_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_safety_veh-inspect_sch-bus_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_511_511.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_air_airport-info_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_air_pilot-info_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_bike_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_ped_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_pub-transit_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_rail_amtrak.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_rail_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_rail_mars.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_rail_mis.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_comm-hwys_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_flexlane_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_hwy-maps_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_out-of-state_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_parknride_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_rest-areas_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_rideshare_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_rustic-roads_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_road_scenic-ways_default.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_water_cruise.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/Pages_travel_water_ferries.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/pages_dmv_license-drvs_rcd-crsh-rpt_auto-insurance.aspx.html',\n",
       " 'gs://lramsey-dev-downloads-test-vertex-search-6/pages_online-srvcs_external_dmv.aspx.html']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs_to_be_processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5439947-e994-4798-a3e6-83ddd3cf8999",
   "metadata": {},
   "outputs": [],
   "source": [
    "operation = create_data_store(q_engine, project_id, data_store_id)\n",
    "wait_for_operation(operation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fc7c0c3-e4d2-482c-a93c-84ddf228f21f",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = discoveryengine.DocumentServiceClient()\n",
    "parent = client.branch_path(\n",
    "    project=project_id,\n",
    "    location=location,\n",
    "    data_store=data_store_id,\n",
    "    branch=\"default_branch\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe6153f0-a916-4337-bb39-261e218539f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdac0319-64b9-4ccc-9055-738f10be0b14",
   "metadata": {},
   "outputs": [],
   "source": [
    "operation = import_documents_gcs(data_url,\n",
    "                                 docs_to_be_processed,\n",
    "                                 client,\n",
    "                                 parent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b2f53a2-315a-47d6-8e08-8105bfbc5ed5",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Waiting for import operation to complete: {operation.operation.name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "897cca9d-e00c-45b1-90d8-b2044e376347",
   "metadata": {},
   "outputs": [],
   "source": [
    "wait_for_operation(operation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5617ae9-50a3-4b0e-9ee4-38e22431d823",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = discoveryengine.ImportDocumentsMetadata(operation.metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95aa0c3b-611d-4e15-a97d-e308f11f7b59",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5fe2ff9-493e-496b-b002-5c09db9c5527",
   "metadata": {},
   "outputs": [],
   "source": [
    "operation.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd07e585-99d8-4fc4-ac0d-9cbfa888ce0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "operation.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d0b6c47-13e6-4c8a-9831-6c98c6de30c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!gsutil ls -R gs://944045413892_us_import_content/errors16429642499957673123"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0108e1e-1722-48e9-a89c-6f59d52c7fed",
   "metadata": {},
   "outputs": [],
   "source": [
    "if metadata.success_count == len(docs_to_be_processed):\n",
    "  docs_processed = docs_to_be_processed\n",
    "else:\n",
    "  # TODO: build list of documents processed/not processed from results\n",
    "  pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3753869-05af-4376-8ac6-b225331ddbd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "095d23fc-3d5c-4f50-9191-b6d564843bd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "operation = create_search_engine(q_engine, project_id, data_store_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5d6ca75-5fe5-46cc-ba01-93b86a572c27",
   "metadata": {},
   "outputs": [],
   "source": [
    "q_engine.index_id = data_store_id\n",
    "q_engine.update()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9816ab68-cfba-4de3-8d20-e6a266ca003a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from services.query.vertex_search import query_vertex_search, perform_vertex_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cbe6ed2-f8e4-45c3-be63-aed19a903c94",
   "metadata": {},
   "outputs": [],
   "source": [
    "search_query = \"what qualifies as an encounter in medicaid billing?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a89755e-0e9a-4824-b4bf-58b63306b883",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = discoveryengine.SearchServiceClient()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4a13a5b-8ceb-4f99-aa7f-0a2001a6415f",
   "metadata": {},
   "outputs": [],
   "source": [
    "serving_config = client.serving_config_path(\n",
    "    project=project_id,\n",
    "    location=location,\n",
    "    data_store=data_store_id,\n",
    "    serving_config=\"default_config\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "984d8a4d-4f64-42a4-a656-ea09b087e0ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "content_search_spec = discoveryengine.SearchRequest.ContentSearchSpec(\n",
    "    snippet_spec=discoveryengine.SearchRequest.ContentSearchSpec.SnippetSpec(\n",
    "        return_snippet=True\n",
    "    ),\n",
    "    summary_spec=discoveryengine.SearchRequest.ContentSearchSpec.SummarySpec(\n",
    "        summary_result_count=5,\n",
    "        include_citations=True,\n",
    "        ignore_adversarial_query=True,\n",
    "        ignore_non_summary_seeking_query=True,\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "723a8f36-9705-42bc-bd9c-3b2f4b55495e",
   "metadata": {},
   "outputs": [],
   "source": [
    "request = discoveryengine.SearchRequest(\n",
    "    serving_config=serving_config,\n",
    "    query=search_query,\n",
    "    page_size=10,\n",
    "    content_search_spec=content_search_spec,\n",
    "    query_expansion_spec=discoveryengine.SearchRequest.QueryExpansionSpec(\n",
    "        condition=discoveryengine.SearchRequest.QueryExpansionSpec.Condition.AUTO,\n",
    "    ),\n",
    "    spell_correction_spec=discoveryengine.SearchRequest.SpellCorrectionSpec(\n",
    "        mode=discoveryengine.SearchRequest.SpellCorrectionSpec.Mode.AUTO\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2091e2c-0637-4cff-a538-4e9d4b172b4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = client.search(request)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f10418cc-faa0-41a8-baa3-9056b496c979",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(response.results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bbab0e3-c248-4922-bcc9-955d00de05bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = response.results\n",
    "type(results[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acc18167-f4a3-41df-aebb-c2a2189a2fd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "type(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c97cefd-2980-450a-946e-f349b741fe18",
   "metadata": {},
   "outputs": [],
   "source": [
    "document = results[0].document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ad5d8ee-6430-4a84-99ed-e874a60f1652",
   "metadata": {},
   "outputs": [],
   "source": [
    "type(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c1811a2-99a8-4e6e-94ac-adee901d405b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.protobuf.json_format import MessageToDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "181265b6-b603-471e-820e-844929326584",
   "metadata": {},
   "outputs": [],
   "source": [
    "import proto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "554b3407-6330-4f3d-bf6b-502d51f46d71",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_dict = proto.Message.to_dict(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b9035f8-55a8-48a0-a2cc-69b68de5045c",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "283f027e-9056-4728-b6c4-75bb73f6b95b",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_dict[\"derived_struct_data\"][\"snippets\"][0][\"snippet\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0453d433-c4ae-465d-8436-9c1be3a5bc08",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_dict[\"derived_struct_data\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0809819-3599-4ccb-bc55-3a948253762d",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_data = proto.Message.to_dict(document)[\"derived_struct_data\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9d22410-12dd-4aee-9c41-53d1574839f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_data[\"link\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "972e5e45-ad6d-4a69-94b9-84476f20fdc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_data[\"snippets\"][0][\"snippet\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "519d3f71-f9c0-41ab-9530-ab895c40383d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}