Update CDP Mode

Author: mdmintz

examples/cdp_mode/ReadMe.md

@@ -471,6 +471,7 @@ sb.cdp.get_current_url()
 sb.cdp.get_origin()
 sb.cdp.get_html(include_shadow_dom=True)
 sb.cdp.get_page_source(include_shadow_dom=True)
+sb.cdp.get_beautiful_soup(source=None)
 sb.cdp.get_user_agent()
 sb.cdp.get_cookie_string()
 sb.cdp.get_locale_code()

help_docs/cdp_mode_methods.md

@@ -99,6 +99,7 @@ sb.cdp.get_current_url()
 sb.cdp.get_origin()
 sb.cdp.get_html(include_shadow_dom=True)
 sb.cdp.get_page_source(include_shadow_dom=True)
+sb.cdp.get_beautiful_soup(source=None)
 sb.cdp.get_user_agent()
 sb.cdp.get_cookie_string()
 sb.cdp.get_locale_code()

seleniumbase/core/browser_launcher.py

@@ -872,6 +872,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs):
     cdp.get_gui_element_center = CDPM.get_gui_element_center
     cdp.get_html = CDPM.get_html
     cdp.get_page_source = CDPM.get_page_source
+    cdp.get_beautiful_soup = CDPM.get_beautiful_soup
     cdp.get_user_agent = CDPM.get_user_agent
     cdp.get_cookie_string = CDPM.get_cookie_string
     cdp.get_locale_code = CDPM.get_locale_code

seleniumbase/core/sb_cdp.py

@@ -117,10 +117,10 @@ def get(self, url, **kwargs):
         if hasattr(driver, "cdp_base"):
             driver = driver.cdp_base
         load_timeout = 60.0
-        wait_timeout = 30.0
+        wait_timeout = 50.0
         if hasattr(sb_config, "_cdp_proxy") and sb_config._cdp_proxy:
             load_timeout = 90.0
-            wait_timeout = 45.0
+            wait_timeout = 75.0
         try:
             task = self.page.get(url, **kwargs)
             self.loop.run_until_complete(
@@ -1399,6 +1399,20 @@ def get_page_source(self, include_shadow_dom=True):
             )
         return source
 
+    def get_beautiful_soup(self, source=None):
+        """BeautifulSoup is a toolkit for dissecting an HTML document
+        and extracting what you need. It's great for screen-scraping!
+        See: https://www.crummy.com/software/BeautifulSoup/bs4/doc/ """
+        from bs4 import BeautifulSoup
+
+        if not source:
+            with suppress(Exception):
+                self.wait_for_element_visible(
+                    "body", timeout=settings.MINI_TIMEOUT
+                )
+            source = self.get_page_source()
+        return BeautifulSoup(source, "html.parser")
+
     def get_user_agent(self):
         return self.loop.run_until_complete(
             self.page.evaluate("navigator.userAgent")

seleniumbase/undetected/cdp_driver/cdp_util.py

@@ -31,6 +31,7 @@
 PROXY_DIR_LOCK = proxy_helper.PROXY_DIR_LOCK
 EXTENSIONS_DIR = os.path.dirname(os.path.realpath(extensions.__file__))
 AD_BLOCK_ZIP_PATH = os.path.join(EXTENSIONS_DIR, "ad_block.zip")
+DISABLE_CSP_ZIP_PATH = os.path.join(EXTENSIONS_DIR, "disable_csp.zip")
 T = typing.TypeVar("T")
 
 
@@ -598,16 +599,6 @@ async def start(
                 proxy_pass,
                 proxy_scheme,
             )
-    if ad_block:
-        sb_config.ad_block_on = True
-        incognito = False
-        guest = False
-        ad_block_zip = AD_BLOCK_ZIP_PATH
-        ad_block_dir = os.path.join(DOWNLOADS_FOLDER, "ad_block")
-        __unzip_to_new_folder(ad_block_zip, ad_block_dir)
-        extension_dir = __add_chrome_ext_dir(extension_dir, ad_block_dir)
-    if disable_csp:
-        sb_config.disable_csp = True
     if "binary_location" in kwargs and not browser_executable_path:
         browser_executable_path = kwargs["binary_location"]
     if not user_data_dir and "--user-data-dir" in arg_join:
@@ -624,8 +615,8 @@ async def start(
             user_data_dir = udd_string
     if user_data_dir:
         user_data_dir = os.path.abspath(user_data_dir)
+    browser = None
     if not browser_executable_path:
-        browser = None
         if "browser" in kwargs:
             browser = kwargs["browser"]
         if not browser and "--browser" in arg_join:
@@ -675,6 +666,23 @@ async def start(
             sb_config._cdp_browser = "atlas"
         else:
             sb_config._cdp_browser = "chrome"
+    if ad_block:
+        sb_config.ad_block_on = True
+        incognito = False
+        guest = False
+        ad_block_zip = AD_BLOCK_ZIP_PATH
+        ad_block_dir = os.path.join(DOWNLOADS_FOLDER, "ad_block")
+        __unzip_to_new_folder(ad_block_zip, ad_block_dir)
+        extension_dir = __add_chrome_ext_dir(extension_dir, ad_block_dir)
+    if disable_csp:
+        sb_config.disable_csp = True
+        if not incognito and not guest:
+            disable_csp_zip = DISABLE_CSP_ZIP_PATH
+            disable_csp_dir = os.path.join(DOWNLOADS_FOLDER, "disable_csp")
+            __unzip_to_new_folder(disable_csp_zip, disable_csp_dir)
+            extension_dir = __add_chrome_ext_dir(
+                extension_dir, disable_csp_dir
+            )
     sb_config.incognito = incognito
     sb_config.guest_mode = guest
     if not config: