in packages/blueprints/gen-ai-chatbot/static-assets/chatbot-genai-components/backend/python/embedding/loaders/playwright.py [0:0]
def evaluate(self, page: "Page", browser: "Browser", response: "Response") -> str:
"""Synchronously process the HTML content of the page."""
from unstructured.partition.html import partition_html
for selector in self.remove_selectors or []:
elements = page.locator(selector).all()
for element in elements:
if element.is_visible():
element.evaluate("element => element.remove()")
# Delay to wait for the page to load.
time.sleep(self.delay_sec)
page_source = page.content()
elements = partition_html(text=page_source)
return "\n\n".join([str(el) for el in elements])