Using the Windows system API we can hide any window using the SW_HIDE method. This is particularly good because we can use Chrome but hide it from the UI and have an increased success rate when loading complicated websites. The reason for the success rate increase is headless often leaves breadcrumbs that “headless” is being run, even when using stealth plugins etc, it is a constant cat and mouse game and requires a high level of skill to find these issues and then create a patch for them.
So by allowing Chrome to render a GUI but actually hide it we bypass these issues and increase scraping success rate.
Here is an example we had:
1st test with hidden GUI:
success: 44, failure: 0, error: 0
2nd test with headless:
success: 5, failure: 35, error: 0
maybe IP got exhausted, so I ran again the hidden GUI test:
success: 40, failure: 0, error: 0
As you can see the results were quite astonishing that simply having the UI hidden massively allowed us to increase success rates per IP.
One problem that I warn we only partially solved is the fact that while you can launch an application and have its PID hidden automatically the problem is Chrome then starts sub processes with other PIDs and this doesn’t hide them.
So our solution was to iterate over Chrome handle names and hide them. Additionally setting it off screen (that can cause issues as well) helps reduce a “flicker” that you will notice when launching Chrome.
Below is Python code you can easily run on Windows.
import subprocess
import win32gui
import win32con
import win32process
import time
import os
import psutil
import json
# Path to Chrome executable (adjust if needed)
chrome_path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
# Custom Chrome profile settings
custom_user_data_dir = r"C:\Users\YourUsername\AppData\Local\Google\Chrome\CustomProfile"
profile_name = "Profile 1"
# Ensure the custom user data directory exists
if not os.path.exists(custom_user_data_dir):
os.makedirs(custom_user_data_dir)
# Clear Chrome's crash flag for the custom profile
prefs_file = os.path.join(custom_user_data_dir, profile_name, "Preferences")
if os.path.exists(prefs_file):
try:
with open(prefs_file, 'r') as f:
prefs = json.load(f)
prefs['profile']['exit_type'] = 'Normal'
prefs['profile']['exited_cleanly'] = True
with open(prefs_file, 'w') as f:
json.dump(prefs, f)
except Exception as e:
print(f"Warning: Could not modify Preferences file: {e}")
# Command-line arguments
args = [
"--new-window",
"https://example.com",
"--no-default-browser-check",
"--disable-session-crashed-bubble",
f"--user-data-dir={custom_user_data_dir}",
f"--profile-directory={profile_name}",
"--window-position=-32000,-32000",
"--window-size=1,1"
]
# Set up STARTUPINFO to start Chrome hidden
si = subprocess.STARTUPINFO()
si.dwFlags = subprocess.STARTF_USESHOWWINDOW
si.wShowWindow = win32con.SW_HIDE
# Launch Chrome
process = subprocess.Popen([chrome_path] + args, startupinfo=si)
main_pid = process.pid
# Get all child process IDs for the new Chrome instance
def get_child_pids(parent_pid):
pids = {parent_pid}
try:
parent = psutil.Process(parent_pid)
for child in parent.children(recursive=True):
pids.add(child.pid)
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
return pids
# Callback function to handle Chrome windows for the new process
def enum_windows_callback(hwnd, results):
try:
_, pid = win32process.GetWindowThreadProcessId(hwnd)
if pid in results['pids'] and "Google Chrome" in win32gui.GetWindowText(hwnd) and win32gui.IsWindow(hwnd):
results['windows'].append(hwnd)
except Exception:
pass
# Function to hide window, modify styles, and move off-screen
def hide_window(hwnd):
# Remove WS_VISIBLE style
style = win32gui.GetWindowLong(hwnd, win32con.GWL_STYLE)
style &= ~win32con.WS_VISIBLE
win32gui.SetWindowLong(hwnd, win32con.GWL_STYLE, style)
# Add WS_EX_TOOLWINDOW to exclude from taskbar
ex_style = win32gui.GetWindowLong(hwnd, win32con.GWL_EXSTYLE)
ex_style |= win32con.WS_EX_TOOLWINDOW
ex_style &= ~win32con.WS_EX_APPWINDOW
win32gui.SetWindowLong(hwnd, win32con.GWL_EXSTYLE, ex_style)
# Move window off-screen
win32gui.SetWindowPos(hwnd, None, -32000, -32000, 0, 0, win32con.SWP_NOSIZE | win32con.SWP_NOZORDER)
# Hide window
win32gui.ShowWindow(hwnd, win32con.SW_HIDE)
# Verify
x, y, _, _ = win32gui.GetWindowRect(hwnd)
return not win32gui.IsWindowVisible(hwnd) or x <= -32000
# Poll for Chrome windows belonging to the new process
start_time = time.time()
max_duration = 10 # 10 seconds for delayed windows
poll_interval = 0.002 # Check every 2ms
max_retries = 5 # Retry up to 5 times per window
child_pids = get_child_pids(main_pid)
while time.time() - start_time < max_duration:
results = {'pids': child_pids, 'windows': []}
win32gui.EnumWindows(enum_windows_callback, results)
windows = results['windows']
for hwnd in windows:
retries = 0
while retries < max_retries:
if hide_window(hwnd):
break
retries += 1
time.sleep(0.002)
if retries >= max_retries:
# Final attempt
style = win32gui.GetWindowLong(hwnd, win32con.GWL_STYLE)
style &= ~win32con.WS_VISIBLE
win32gui.SetWindowLong(hwnd, win32con.GWL_STYLE, style)
ex_style = win32gui.GetWindowLong(hwnd, win32con.GWL_EXSTYLE)
ex_style |= win32con.WS_EX_TOOLWINDOW
ex_style &= ~win32con.WS_EX_APPWINDOW
win32gui.SetWindowLong(hwnd, win32con.GWL_EXSTYLE, ex_style)
win32gui.SetWindowPos(hwnd, None, -32000, -32000, 0, 0, win32con.SWP_NOSIZE | win32con.SWP_NOZORDER)
win32gui.ShowWindow(hwnd, win32con.SW_HIDE)
if windows:
still_visible = any(win32gui.IsWindowVisible(hwnd) and win32gui.GetWindowRect(hwnd)[0] > -32000 for hwnd in windows)
if not still_visible:
break
time.sleep(poll_interval)
# Chrome is now running with the specified profile, window hidden, and no taskbar icon
# You can interact with the process via process.pid, process.wait(), etc.
Here is code you can run that uses this method with Selenium
import subprocess
import win32gui
import win32con
import win32process
import time
import os
import psutil
import json
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# Resolve current user's profile directory and set a dedicated custom profile
current_user = os.environ.get('USERNAME') or 'Default'
custom_user_data_dir = os.path.join(
os.path.expandvars(r"C:\\Users\\%USERNAME%\\AppData\\Local\\Google\\Chrome"),
"CustomProfile",
)
profile_name = "Profile 1"
# Ensure the custom user data directory exists
if not os.path.exists(custom_user_data_dir):
os.makedirs(custom_user_data_dir)
# Clear Chrome's crash flag for the custom profile
prefs_file = os.path.join(custom_user_data_dir, profile_name, "Preferences")
if os.path.exists(prefs_file):
try:
with open(prefs_file, 'r') as f:
prefs = json.load(f)
if 'profile' in prefs:
prefs['profile']['exit_type'] = 'Normal'
prefs['profile']['exited_cleanly'] = True
with open(prefs_file, 'w') as f:
json.dump(prefs, f)
except Exception as e:
print(f"Warning: Could not modify Preferences file: {e}")
# Selenium Chrome options (we will NOT use headless; we'll hide the GUI window ourselves)
chrome_options = Options()
chrome_options.add_argument("--new-window")
chrome_options.add_argument("--no-default-browser-check")
chrome_options.add_argument("--disable-session-crashed-bubble")
chrome_options.add_argument(f"--user-data-dir={custom_user_data_dir}")
chrome_options.add_argument(f"--profile-directory={profile_name}")
# Start off-screen and tiny; we'll additionally remove visibility/taskbar styles below
chrome_options.add_argument("--window-position=-32000,-32000")
chrome_options.add_argument("--window-size=1,1")
chrome_options.set_capability('goog:loggingPrefs', {'performance': 'ALL'})
# Launch Chrome via Selenium (Chromedriver managed automatically)
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service, options=chrome_options)
# Capture Chromedriver PID and Chrome child PIDs
def get_child_pids(parent_pid):
pids = {parent_pid}
try:
parent = psutil.Process(parent_pid)
for child in parent.children(recursive=True):
pids.add(child.pid)
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
return pids
def enum_windows_callback(hwnd, results):
try:
_, pid = win32process.GetWindowThreadProcessId(hwnd)
title = win32gui.GetWindowText(hwnd)
if pid in results['pids'] and win32gui.IsWindow(hwnd) and title:
results['windows'].append(hwnd)
except Exception:
pass
def hide_window(hwnd):
style = win32gui.GetWindowLong(hwnd, win32con.GWL_STYLE)
style &= ~win32con.WS_VISIBLE
win32gui.SetWindowLong(hwnd, win32con.GWL_STYLE, style)
ex_style = win32gui.GetWindowLong(hwnd, win32con.GWL_EXSTYLE)
ex_style |= win32con.WS_EX_TOOLWINDOW
ex_style &= ~win32con.WS_EX_APPWINDOW
win32gui.SetWindowLong(hwnd, win32con.GWL_EXSTYLE, ex_style)
win32gui.SetWindowPos(
hwnd,
None,
-32000,
-32000,
0,
0,
win32con.SWP_NOSIZE | win32con.SWP_NOZORDER | win32con.SWP_NOACTIVATE,
)
win32gui.ShowWindow(hwnd, win32con.SW_HIDE)
x, y, _, _ = win32gui.GetWindowRect(hwnd)
return not win32gui.IsWindowVisible(hwnd) or x <= -32000
# Poll for Chrome windows belonging to Chromedriver's process tree and hide them
start_time = time.time()
max_duration = 10
poll_interval = 0.01
max_retries = 5
# Chromedriver process PID
driver_pid = None
try:
if driver.service and driver.service.process:
driver_pid = driver.service.process.pid
except Exception:
driver_pid = None
if driver_pid is None:
# Fallback: try to detect chromedriver by name if service.process not available
for proc in psutil.process_iter(attrs=["pid", "name"]):
try:
if proc.info["name"] and "chromedriver" in proc.info["name"].lower():
driver_pid = proc.info["pid"]
break
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
child_pids = get_child_pids(driver_pid) if driver_pid else set()
while time.time() - start_time < max_duration:
# Refresh child PIDs in case Chrome processes spawn slightly later
if driver_pid:
child_pids = get_child_pids(driver_pid)
results = {'pids': child_pids, 'windows': []}
win32gui.EnumWindows(enum_windows_callback, results)
windows = results['windows']
for hwnd in windows:
retries = 0
while retries < max_retries:
if hide_window(hwnd):
break
retries += 1
time.sleep(0.01)
if retries >= max_retries:
# Final attempt (idempotent)
hide_window(hwnd)
if windows:
still_visible = any(
win32gui.IsWindowVisible(hwnd) and win32gui.GetWindowRect(hwnd)[0] > -32000
for hwnd in windows
)
if not still_visible:
break
time.sleep(poll_interval)
# Navigate to the requested URL using the hidden Chrome GUI
driver.get("https://www.example.com")
# Keep driver reference available to caller; comment out the next line if you need it to persist
# driver.quit()
# Wait for document ready state to ensure page is fully loaded
try:
WebDriverWait(driver, 15).until(
lambda d: d.execute_script("return document.readyState") == "complete"
)
except Exception:
pass
# Extract HTTP status code for the main document from performance logs
def try_get_main_status_code(target_url):
try:
logs = driver.get_log('performance')
except Exception:
return None
status = None
for entry in logs:
try:
message = json.loads(entry.get('message', '{}'))
msg = message.get('message', {})
method = msg.get('method')
params = msg.get('params', {})
if method == 'Network.responseReceived':
response = params.get('response', {})
url = response.get('url')
rtype = params.get('type')
if (url == target_url) or (rtype == 'Document' and target_url in url):
status = response.get('status')
except Exception:
continue
return status
status_code = try_get_main_status_code("https://www.example.com")
# Print outputs
print(f"HTTP status: {status_code if status_code is not None else 'Unknown'}")
try:
html = driver.page_source
print(html)
except Exception as e:
print(f"Failed to get page source: {e}")