Fix Selenium Chrome session issues by simplifying options and adding error handling
This commit is contained in:
1 parent
bf7aa42460
commit
8393e5d659
1 file changed
+8
-8
@@ -48,20 +48,15 @@ def scrape_recently_played():
|
|||||||
options.add_argument("--no-sandbox")
|
options.add_argument("--no-sandbox")
|
||||||
options.add_argument("--disable-dev-shm-usage")
|
options.add_argument("--disable-dev-shm-usage")
|
||||||
options.add_argument("--disable-gpu")
|
options.add_argument("--disable-gpu")
|
||||||
options.add_argument("--remote-debugging-port=9222")
|
options.add_argument("--remote-debugging-port=0")
|
||||||
options.add_argument("--no-first-run")
|
|
||||||
options.add_argument("--disable-extensions")
|
|
||||||
|
|
||||||
# Set binary location based on OS
|
# Set binary location based on OS
|
||||||
if platform.system() == 'Windows':
|
if platform.system() == 'Windows':
|
||||||
options.binary_location = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
|
options.binary_location = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
|
||||||
options.add_argument("--user-data-dir=C:\\temp\\chromium")
|
|
||||||
elif platform.system() == 'Darwin': # macOS
|
elif platform.system() == 'Darwin': # macOS
|
||||||
options.binary_location = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
|
options.binary_location = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
|
||||||
options.add_argument("--user-data-dir=/tmp/chromium")
|
|
||||||
else: # Linux
|
else: # Linux
|
||||||
options.binary_location = "/usr/bin/chromium-browser"
|
options.binary_location = "/usr/bin/chromium-browser"
|
||||||
options.add_argument("--user-data-dir=/tmp/chromium")
|
|
||||||
|
|
||||||
service = Service()
|
service = Service()
|
||||||
driver = webdriver.Chrome(service=service, options=options)
|
driver = webdriver.Chrome(service=service, options=options)
|
||||||
@@ -69,17 +64,22 @@ def scrape_recently_played():
|
|||||||
driver.get(url)
|
driver.get(url)
|
||||||
|
|
||||||
# Wait for initial load
|
# Wait for initial load
|
||||||
time.sleep(5)
|
time.sleep(10)
|
||||||
|
|
||||||
# Click "View More" button if exists
|
# Click "View More" button if exists
|
||||||
try:
|
try:
|
||||||
more_button = driver.find_element(By.ID, "moreSongs")
|
more_button = driver.find_element(By.ID, "moreSongs")
|
||||||
more_button.click()
|
more_button.click()
|
||||||
time.sleep(5) # Wait for more songs to load
|
time.sleep(10) # Wait for more songs to load
|
||||||
except:
|
except:
|
||||||
pass # Button not found or already loaded
|
pass # Button not found or already loaded
|
||||||
|
|
||||||
|
try:
|
||||||
html = driver.page_source
|
html = driver.page_source
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Failed to get page source for {source}: {e}")
|
||||||
|
html = ""
|
||||||
|
|
||||||
driver.quit()
|
driver.quit()
|
||||||
|
|
||||||
soup = BeautifulSoup(html, 'html.parser')
|
soup = BeautifulSoup(html, 'html.parser')
|
||||||
|
|||||||
Reference in new issue
Block a user