Fix song scraping - use proper HTML structure parsing
This commit is contained in:
nmemmert committed
1 parent
b50b430a5f
commit
e762c4252d
4 files changed
+122
-83
No files matched your search
@@ -141,3 +141,24 @@ https://www.amazon.com/s?k=You%20Danny%20%20Gokey%20I%20%20Got%20&i=digital-musi
|
|||||||
You've Done Tasha Layton - Look What
|
You've Done Tasha Layton - Look What
|
||||||
https://www.amazon.com/s?k=You%27ve%20%20Done%20Tasha%20%20Layton%20Look%20%20What%20&i=digital-music
|
https://www.amazon.com/s?k=You%27ve%20%20Done%20Tasha%20%20Layton%20Look%20%20What%20&i=digital-music
|
||||||
|
|
||||||
|
Zach Williams - Jesus Loves
|
||||||
|
https://www.amazon.com/s?k=Zach%20Williams%20Jesus%20Loves&i=digital-music
|
||||||
|
|
||||||
|
Zach Williams - Jesus Loves
|
||||||
|
https://www.amazon.com/s?k=Zach%20%20Williams%20Jesus%20%20Loves&i=digital-music
|
||||||
|
|
||||||
|
Am They - Scars I
|
||||||
|
https://www.amazon.com/s?k=Am%20%20They%20Scars%20I%20&i=digital-music
|
||||||
|
|
||||||
|
Church Sing Tauren Wells - Let The
|
||||||
|
https://www.amazon.com/s?k=Church%20%20Sing%20Tauren%20%20Wells%20Let%20%20The%20&i=digital-music
|
||||||
|
|
||||||
|
You Danny Gokey - I Got
|
||||||
|
https://www.amazon.com/s?k=You%20Danny%20%20Gokey%20I%20%20Got%20&i=digital-music
|
||||||
|
|
||||||
|
Cochren & Co. - Church (Take Me Back)
|
||||||
|
https://www.amazon.com/s?k=Cochren%20%26%20Co.%20Church%20%28Take%20Me%20Back%29&i=digital-music
|
||||||
|
|
||||||
|
Blessing Offor - Your Love
|
||||||
|
https://www.amazon.com/s?k=Blessing%20Offor%20Your%20Love&i=digital-music
|
||||||
|
|
||||||
+12
@@ -4,3 +4,15 @@
|
|||||||
2025-11-17 12:42:52.626588: Starting Journey FM Playlist app...
|
2025-11-17 12:42:52.626588: Starting Journey FM Playlist app...
|
||||||
2025-11-17 12:42:52.626688: DISPLAY set, initializing GUI
|
2025-11-17 12:42:52.626688: DISPLAY set, initializing GUI
|
||||||
2025-11-17 12:42:52.761510: App initialized, starting event loop
|
2025-11-17 12:42:52.761510: App initialized, starting event loop
|
||||||
|
2025-11-17 12:56:44.139926: Starting Journey FM Playlist app...
|
||||||
|
2025-11-17 12:56:44.140042: DISPLAY set, initializing GUI
|
||||||
|
2025-11-17 12:56:44.381412: App initialized, starting event loop
|
||||||
|
2025-11-17 13:01:57.291192: Starting Journey FM Playlist app...
|
||||||
|
2025-11-17 13:01:57.291370: DISPLAY set, initializing GUI
|
||||||
|
2025-11-17 13:01:57.440291: App initialized, starting event loop
|
||||||
|
2025-11-17 13:09:29.113472: Starting Journey FM Playlist app...
|
||||||
|
2025-11-17 13:09:29.113563: DISPLAY set, initializing GUI
|
||||||
|
2025-11-17 13:09:29.245236: App initialized, starting event loop
|
||||||
|
2025-11-17 13:12:14.103261: Starting Journey FM Playlist app...
|
||||||
|
2025-11-17 13:12:14.103373: DISPLAY set, initializing GUI
|
||||||
|
2025-11-17 13:12:14.295725: App initialized, starting event loop
|
||||||
@@ -13,6 +13,8 @@ from selenium.webdriver.chrome.options import Options
|
|||||||
from selenium.webdriver.chrome.service import Service
|
from selenium.webdriver.chrome.service import Service
|
||||||
from webdriver_manager.chrome import ChromeDriverManager
|
from webdriver_manager.chrome import ChromeDriverManager
|
||||||
from selenium.webdriver.common.by import By
|
from selenium.webdriver.common.by import By
|
||||||
|
from selenium.webdriver.support.ui import WebDriverWait
|
||||||
|
from selenium.webdriver.support import expected_conditions as EC
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
import os
|
import os
|
||||||
from plexapi.myplex import MyPlexAccount
|
from plexapi.myplex import MyPlexAccount
|
||||||
@@ -31,103 +33,107 @@ SERVER_IP = '172.16.16.106' # Your local server IP
|
|||||||
PLAYLIST_NAME = 'Journey FM Recently Played'
|
PLAYLIST_NAME = 'Journey FM Recently Played'
|
||||||
|
|
||||||
def scrape_recently_played():
|
def scrape_recently_played():
|
||||||
"""Scrape recently played songs from multiple stations using Selenium"""
|
"""Scrape recently played songs from Journey FM and Spirit FM"""
|
||||||
import platform
|
options = Options()
|
||||||
|
options.add_argument("--headless=new")
|
||||||
|
options.add_argument("--no-sandbox")
|
||||||
|
options.add_argument("--disable-dev-shm-usage")
|
||||||
|
options.add_argument("--disable-gpu")
|
||||||
|
options.add_argument("--window-size=1920,1080")
|
||||||
|
options.binary_location = "/usr/bin/google-chrome-stable"
|
||||||
|
|
||||||
urls = [
|
service = Service(ChromeDriverManager().install())
|
||||||
('https://www.myjourneyfm.com/recently-played/', 'Journey FM'),
|
|
||||||
('https://spiritfm.com/spiritfm-recently-played/', 'Spirit FM')
|
try:
|
||||||
]
|
driver = webdriver.Chrome(service=service, options=options)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Failed to start Chrome: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
all_songs = []
|
all_songs = []
|
||||||
seen = set() # To avoid duplicates
|
seen = set()
|
||||||
|
|
||||||
for url, source in urls:
|
# Journey FM - uses <strong> for title, normal text for artist
|
||||||
options = Options()
|
try:
|
||||||
options.add_argument("--headless")
|
print("Scraping Journey FM...")
|
||||||
options.add_argument("--no-sandbox")
|
driver.get('https://www.myjourneyfm.com/recently-played/')
|
||||||
options.add_argument("--disable-dev-shm-usage")
|
time.sleep(5)
|
||||||
options.add_argument("--disable-gpu")
|
|
||||||
|
|
||||||
service = Service(ChromeDriverManager().install())
|
|
||||||
driver = webdriver.Chrome(service=service, options=options)
|
|
||||||
|
|
||||||
driver.get(url)
|
|
||||||
|
|
||||||
# Wait for initial load
|
|
||||||
time.sleep(10)
|
|
||||||
|
|
||||||
# Click "View More" button if exists
|
# Click "View More" button if exists
|
||||||
try:
|
try:
|
||||||
more_button = driver.find_element(By.ID, "moreSongs")
|
more_button = driver.find_element(By.ID, "moreSongs")
|
||||||
more_button.click()
|
more_button.click()
|
||||||
time.sleep(10) # Wait for more songs to load
|
time.sleep(5)
|
||||||
except:
|
except:
|
||||||
pass # Button not found or already loaded
|
pass
|
||||||
|
|
||||||
try:
|
soup = BeautifulSoup(driver.page_source, 'html.parser')
|
||||||
html = driver.page_source
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Failed to get page source for {source}: {e}")
|
|
||||||
html = ""
|
|
||||||
|
|
||||||
driver.quit()
|
# Find all song items using the proper class structure
|
||||||
|
song_items = soup.find_all('div', class_='rp-item')
|
||||||
|
|
||||||
soup = BeautifulSoup(html, 'html.parser')
|
for item in song_items:
|
||||||
|
try:
|
||||||
songs = []
|
title_elem = item.find('h5', class_='song-title')
|
||||||
# Find elements (div, li) that contain time in their text
|
artist_elem = item.find('p', class_='song-artist')
|
||||||
song_elements = soup.find_all(lambda tag: tag.name in ['div', 'li', 'p'] and re.search(r'\d+:\d+', tag.get_text()))
|
|
||||||
for element in song_elements:
|
if title_elem and artist_elem:
|
||||||
spans = element.find_all('span')
|
title = title_elem.get_text().strip()
|
||||||
if len(spans) >= 2:
|
artist = artist_elem.get_text().strip()
|
||||||
title = spans[0].get_text().strip()
|
|
||||||
artist = spans[1].get_text().strip()
|
if title and artist:
|
||||||
else:
|
key = (title.lower(), artist.lower())
|
||||||
# Check for strong
|
if key not in seen:
|
||||||
strong = element.find('strong')
|
all_songs.append({'title': title, 'artist': artist, 'source': 'Journey FM'})
|
||||||
if strong:
|
seen.add(key)
|
||||||
title = strong.get_text().strip()
|
print(f" Found: {title} by {artist}")
|
||||||
full_text = element.get_text().strip()
|
except Exception as e:
|
||||||
match = re.search(r'(\d+:\d+(?: [AP]M)?)', full_text)
|
continue
|
||||||
if match:
|
|
||||||
time_part = match.group(1)
|
except Exception as e:
|
||||||
before_time = full_text.split(time_part)[0].strip()
|
print(f"Error scraping Journey FM: {e}")
|
||||||
artist = before_time.replace(title, '').strip()
|
|
||||||
else:
|
|
||||||
artist = full_text.replace(title, '').strip()
|
|
||||||
artist = artist.strip()
|
|
||||||
else:
|
|
||||||
# Fallback to text parsing
|
|
||||||
full_text = element.get_text().strip()
|
|
||||||
match = re.search(r'(\d+:\d+(?: [AP]M)?)', full_text)
|
|
||||||
if match:
|
|
||||||
play_time = match.group(1)
|
|
||||||
before = full_text.split(play_time)[0].strip()
|
|
||||||
if ' by ' in before:
|
|
||||||
title, artist = before.split(' by ', 1)
|
|
||||||
title = title.strip()
|
|
||||||
artist = artist.strip()
|
|
||||||
elif ' - ' in before:
|
|
||||||
title, artist = before.split(' - ', 1)
|
|
||||||
title = title.strip()
|
|
||||||
artist = artist.strip()
|
|
||||||
else:
|
|
||||||
parts = re.findall(r'[A-Z][^A-Z]*', before)
|
|
||||||
if len(parts) >= 4:
|
|
||||||
title = ' '.join(parts[:2])
|
|
||||||
artist = ' '.join(parts[2:])
|
|
||||||
else:
|
|
||||||
title = ' '.join(parts[:len(parts)//2])
|
|
||||||
artist = ' '.join(parts[len(parts)//2:])
|
|
||||||
if title and artist:
|
|
||||||
songs.append({'artist': artist, 'title': title, 'source': source}) # Add to all_songs, avoiding duplicates
|
|
||||||
for song in songs:
|
|
||||||
key = (song['title'].lower(), song['artist'].lower())
|
|
||||||
if key not in seen:
|
|
||||||
all_songs.append(song)
|
|
||||||
seen.add(key)
|
|
||||||
|
|
||||||
|
# Spirit FM - uses Title - Artist format
|
||||||
|
try:
|
||||||
|
print("Scraping Spirit FM...")
|
||||||
|
driver.get('https://spiritfm.com/spiritfm-recently-played/')
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
soup = BeautifulSoup(driver.page_source, 'html.parser')
|
||||||
|
|
||||||
|
# Find all playlist items
|
||||||
|
items = soup.find_all(lambda tag: tag.name in ['div', 'li'] and re.search(r'\d+:\d+\s+[AP]M', tag.get_text()))
|
||||||
|
|
||||||
|
for item in items:
|
||||||
|
try:
|
||||||
|
text = item.get_text().strip()
|
||||||
|
|
||||||
|
# Remove time
|
||||||
|
time_match = re.search(r'\d+:\d+\s+[AP]M', text)
|
||||||
|
if time_match:
|
||||||
|
before_time = text.split(time_match.group())[0].strip()
|
||||||
|
|
||||||
|
# Spirit FM uses dash separator
|
||||||
|
if ' - ' in before_time:
|
||||||
|
parts = before_time.split(' - ', 1)
|
||||||
|
title = parts[0].strip()
|
||||||
|
artist = parts[1].strip()
|
||||||
|
|
||||||
|
if title and artist:
|
||||||
|
key = (title.lower(), artist.lower())
|
||||||
|
if key not in seen:
|
||||||
|
all_songs.append({'title': title, 'artist': artist, 'source': 'Spirit FM'})
|
||||||
|
seen.add(key)
|
||||||
|
print(f" Found: {title} by {artist}")
|
||||||
|
except Exception as e:
|
||||||
|
continue
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error scraping Spirit FM: {e}")
|
||||||
|
|
||||||
|
driver.quit()
|
||||||
|
|
||||||
|
print(f"\nTotal unique songs found: {len(all_songs)}")
|
||||||
return all_songs
|
return all_songs
|
||||||
|
|
||||||
def normalize_string(s):
|
def normalize_string(s):
|
||||||
|
|||||||
Binary file not shown.
Reference in new issue
Block a user