Update CDP Mode

Author: mdmintz

help_docs/cdp_mode_methods.md

@@ -305,6 +305,8 @@ await tab.close()
 await tab.scroll_down(amount=25)
 await tab.scroll_up(amount=25)
 await tab.wait_for(selector="", text="", timeout=10)
+await tab.set_attributes(selector, attribute, value)
+await tab.internalize_links()
 await tab.download_file(url, filename=None)
 await tab.save_screenshot(
     filename="auto", format="png", full_page=False)

seleniumbase/core/browser_launcher.py

@@ -2738,7 +2738,6 @@ def _set_chrome_options(
     chrome_options.add_argument("--disable-save-password-bubble")
     chrome_options.add_argument("--disable-single-click-autofill")
     chrome_options.add_argument("--allow-file-access-from-files")
-    chrome_options.add_argument("--disable-component-update")
     chrome_options.add_argument("--disable-prompt-on-repost")
     chrome_options.add_argument("--dns-prefetch-disable")
     chrome_options.add_argument("--disable-translate")
@@ -4793,7 +4792,6 @@ def get_local_driver(
         if devtools and not headless:
             edge_options.add_argument("--auto-open-devtools-for-tabs")
         edge_options.add_argument("--allow-file-access-from-files")
-        edge_options.add_argument("--disable-component-update")
         edge_options.add_argument("--allow-insecure-localhost")
         edge_options.add_argument("--allow-running-insecure-content")
         if user_agent:

seleniumbase/core/sb_cdp.py

@@ -1664,13 +1664,15 @@ def set_attributes(self, selector, attribute, value):
         css_selector = self.__convert_to_css_if_xpath(selector)
         css_selector = re.escape(css_selector)  # Add "\\" to special chars
         css_selector = js_utils.escape_quotes_if_needed(css_selector)
-        js_code = """var $elements = document.querySelectorAll('%s');
-                  var index = 0, length = $elements.length;
-                  for(; index < length; index++){
-                  $elements[index].setAttribute('%s','%s');}""" % (
-            css_selector,
-            attribute,
-            value,
+        js_code = (
+            """var $elements = document.querySelectorAll('%s');
+            var index = 0, length = $elements.length;
+            for(; index < length; index++){
+            $elements[index].setAttribute('%s','%s');}""" % (
+                css_selector,
+                attribute,
+                value,
+            )
         )
         with suppress(Exception):
             self.loop.run_until_complete(self.page.evaluate(js_code))
@@ -1965,8 +1967,8 @@ def _on_a_cf_turnstile_page(self, source=None):
             return True
         return False
 
-    def _on_a_g_recaptcha_page(self, source=None):
-        time.sleep(0.4)
+    def _on_a_g_recaptcha_page(self, *args, **kwargs):
+        time.sleep(0.4)  # reCAPTCHA may need a moment to appear
         self.loop.run_until_complete(self.page.wait())
         source = self.get_page_source()
         if (

seleniumbase/undetected/cdp_driver/browser.py

@@ -580,7 +580,7 @@ async def start(self=None) -> Browser:
             )  # noqa
         exe = self.config.browser_executable_path
         params = self.config()
-        logger.info(
+        logger.debug(
             "Starting\n\texecutable :%s\n\narguments:\n%s",
             exe,
             "\n\t".join(params),
@@ -636,7 +636,7 @@ async def start(self=None) -> Browser:
             self.info.webSocketDebuggerUrl, browser=self
         )
         if self.config.autodiscover_targets:
-            logger.info("Enabling autodiscover targets")
+            logger.debug("Enabling autodiscover targets")
             self.connection.handlers[cdp.target.TargetInfoChanged] = [
                 self._handle_target_update
             ]
@@ -863,15 +863,15 @@ def stop(self):
         for _ in range(3):
             try:
                 self._process.terminate()
-                logger.info(
+                logger.debug(
                     "Terminated browser with pid %d successfully."
                     % self._process.pid
                 )
                 break
             except (Exception,):
                 try:
                     self._process.kill()
-                    logger.info(
+                    logger.debug(
                         "Killed browser with pid %d successfully."
                         % self._process.pid
                     )
@@ -880,14 +880,14 @@ def stop(self):
                     try:
                         if hasattr(self, "browser_process_pid"):
                             os.kill(self._process_pid, 15)
-                            logger.info(
+                            logger.debug(
                                 "Killed browser with pid %d "
                                 "using signal 15 successfully."
                                 % self._process.pid
                             )
                             break
                     except (TypeError,):
-                        logger.info("typerror", exc_info=True)
+                        logger.info("TypeError", exc_info=True)
                         pass
                     except (PermissionError,):
                         logger.info(
@@ -896,7 +896,7 @@ def stop(self):
                         )
                         pass
                     except (ProcessLookupError,):
-                        logger.info("Process lookup failure!")
+                        logger.info("ProcessLookupError")
                         pass
                     except (Exception,):
                         raise

seleniumbase/undetected/cdp_driver/config.py

@@ -207,7 +207,6 @@ def __init__(
             "--disable-top-sites",
             "--disable-translate",
             "--dns-prefetch-disable",
-            "--disable-component-update",
             "--disable-renderer-backgrounding",
             "--disable-dev-shm-usage",
         ]

seleniumbase/undetected/cdp_driver/element.py

@@ -372,7 +372,10 @@ async def click_async(self):
         arguments = [cdp.runtime.CallArgument(
             object_id=self._remote_object.object_id
         )]
-        await self.flash_async(0.25)
+        script = 'sessionStorage.getItem("pxsid") !== null;'
+        using_px = await self.tab.evaluate(script)
+        if not using_px:
+            await self.flash_async(0.25)
         await self._tab.send(
             cdp.runtime.call_function_on(
                 "(el) => el.click()",
@@ -501,7 +504,10 @@ async def mouse_click_async(
             logger.warning("Could not calculate box model for %s", self)
             return
         logger.debug("Clicking on location: %.2f, %.2f" % center)
-        asyncio.create_task(self.flash_async(0.25))
+        script = 'sessionStorage.getItem("pxsid") !== null;'
+        using_px = await self.tab.evaluate(script)
+        if not using_px:
+            asyncio.create_task(self.flash_async(0.25))
         asyncio.create_task(
             self._tab.send(
                 cdp.input_.dispatch_mouse_event(
@@ -560,12 +566,15 @@ async def mouse_click_with_offset_async(
             logger.debug("Clicking on location: %.2f, %.2f" % center_pos)
         else:
             logger.debug("Clicking on location: %.2f, %.2f" % (x_pos, y_pos))
-        asyncio.create_task(
-            self.flash_async(
-                x_offset=x_offset - (width / 2),
-                y_offset=y_offset - (height / 2),
-            ),
-        )
+        script = 'sessionStorage.getItem("pxsid") !== null;'
+        using_px = await self.tab.evaluate(script)
+        if not using_px:
+            asyncio.create_task(
+                self.flash_async(
+                    x_offset=x_offset - (width / 2),
+                    y_offset=y_offset - (height / 2),
+                ),
+            )
         asyncio.create_task(
             self._tab.send(
                 cdp.input_.dispatch_mouse_event(

seleniumbase/undetected/cdp_driver/tab.py

@@ -1092,6 +1092,39 @@ async def wait_for(
                 await self.sleep(0.068)
             return item
 
+    async def set_attributes(self, selector, attribute, value):
+        """This method uses JavaScript to set/update a common attribute.
+        All matching selectors from querySelectorAll() are used.
+        Example => (Make all links on a website redirect to Google):
+        self.set_attributes("a", "href", "https://google.com")"""
+        attribute = re.escape(attribute)
+        attribute = js_utils.escape_quotes_if_needed(attribute)
+        value = re.escape(value)
+        value = js_utils.escape_quotes_if_needed(value)
+        if selector.startswith(("/", "./", "(")):
+            with suppress(Exception):
+                selector = js_utils.convert_to_css_selector(selector, "xpath")
+        css_selector = selector
+        css_selector = re.escape(css_selector)  # Add "\\" to special chars
+        css_selector = js_utils.escape_quotes_if_needed(css_selector)
+        js_code = (
+            """var $elements = document.querySelectorAll('%s');
+            var index = 0, length = $elements.length;
+            for(; index < length; index++){
+            $elements[index].setAttribute('%s','%s');}""" % (
+                css_selector,
+                attribute,
+                value,
+            )
+        )
+        with suppress(Exception):
+            await self.evaluate(js_code)
+
+    async def internalize_links(self):
+        """All `target="_blank"` links become `target="_self"`.
+        This prevents those links from opening in a new tab."""
+        await self.set_attributes('[target="_blank"]', "target", "_self")
+
     async def download_file(
         self, url: str, filename: Optional[PathLike] = None
     ):
@@ -1336,8 +1369,8 @@ async def __on_a_cf_turnstile_page(self, source=None):
             return True
         return False
 
-    async def __on_a_g_recaptcha_page(self, source=None):
-        await self.sleep(0.4)
+    async def __on_a_g_recaptcha_page(self, *args, **kwargs):
+        await self.sleep(0.4)  # reCAPTCHA may need a moment to appear
         source = await self.get_html()
         if (
             (