Add scraping from Spirit FM recently played list, combine with Journey FM and deduplicate songs
This commit is contained in:
nmemmert committed
1 parent
7e7754f533
commit
32d87cf0c2
1 file changed
+78
-67
@@ -31,77 +31,88 @@ SERVER_IP = '172.16.16.106' # Your local server IP
|
||||
PLAYLIST_NAME = 'Journey FM Recently Played'
|
||||
|
||||
def scrape_recently_played():
|
||||
"""Scrape recently played songs from myjourneyfm.com using Selenium"""
|
||||
options = Options()
|
||||
options.add_argument("--headless")
|
||||
options.add_argument("--no-sandbox")
|
||||
options.add_argument("--disable-dev-shm-usage")
|
||||
options.add_argument("--disable-gpu")
|
||||
options.add_argument("--remote-debugging-port=9222")
|
||||
options.add_argument("--user-data-dir=/tmp/chromium")
|
||||
options.add_argument("--no-first-run")
|
||||
options.add_argument("--disable-extensions")
|
||||
options.binary_location = "/usr/bin/chromium-browser"
|
||||
service = Service()
|
||||
driver = webdriver.Chrome(service=service, options=options)
|
||||
"""Scrape recently played songs from multiple stations using Selenium"""
|
||||
urls = [
|
||||
'https://www.myjourneyfm.com/recently-played/',
|
||||
'https://spiritfm.com/spiritfm-recently-played/'
|
||||
]
|
||||
|
||||
url = 'https://www.myjourneyfm.com/recently-played/'
|
||||
driver.get(url)
|
||||
all_songs = []
|
||||
seen = set() # To avoid duplicates
|
||||
|
||||
# Wait for initial load
|
||||
time.sleep(5)
|
||||
|
||||
# Click "View More" button if exists
|
||||
try:
|
||||
more_button = driver.find_element(By.ID, "moreSongs")
|
||||
more_button.click()
|
||||
time.sleep(5) # Wait for more songs to load
|
||||
except:
|
||||
pass # Button not found or already loaded
|
||||
|
||||
html = driver.page_source
|
||||
driver.quit()
|
||||
|
||||
# Optional: save HTML to file for debugging (comment out if not needed)
|
||||
# with open('page.html', 'w', encoding='utf-8') as f:
|
||||
# f.write(html)
|
||||
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
songs = []
|
||||
# Find elements (div, li) that contain time in their text
|
||||
song_elements = soup.find_all(lambda tag: tag.name in ['div', 'li', 'p'] and re.search(r'\d+:\d+ [AP]M', tag.get_text()))
|
||||
for element in song_elements:
|
||||
spans = element.find_all('span')
|
||||
if len(spans) >= 3:
|
||||
title = spans[0].get_text().strip()
|
||||
artist = spans[1].get_text().strip()
|
||||
# Ignore time
|
||||
songs.append({'artist': artist, 'title': title})
|
||||
else:
|
||||
# Fallback to text parsing
|
||||
full_text = element.get_text().strip()
|
||||
match = re.search(r'(\d+:\d+ [AP]M)', full_text)
|
||||
if match:
|
||||
play_time = match.group(1)
|
||||
before = full_text.split(play_time)[0].strip()
|
||||
if ' by ' in before:
|
||||
title, artist = before.split(' by ', 1)
|
||||
title = title.strip()
|
||||
artist = artist.strip()
|
||||
else:
|
||||
# Split on capital letters
|
||||
parts = re.findall(r'[A-Z][^A-Z]*', before)
|
||||
if len(parts) == 4:
|
||||
title = ' '.join(parts[:2])
|
||||
artist = ' '.join(parts[2:])
|
||||
for url in urls:
|
||||
options = Options()
|
||||
options.add_argument("--headless")
|
||||
options.add_argument("--no-sandbox")
|
||||
options.add_argument("--disable-dev-shm-usage")
|
||||
options.add_argument("--disable-gpu")
|
||||
options.add_argument("--remote-debugging-port=9222")
|
||||
options.add_argument("--user-data-dir=/tmp/chromium")
|
||||
options.add_argument("--no-first-run")
|
||||
options.add_argument("--disable-extensions")
|
||||
options.binary_location = "/usr/bin/chromium-browser"
|
||||
service = Service()
|
||||
driver = webdriver.Chrome(service=service, options=options)
|
||||
|
||||
driver.get(url)
|
||||
|
||||
# Wait for initial load
|
||||
time.sleep(5)
|
||||
|
||||
# Click "View More" button if exists
|
||||
try:
|
||||
more_button = driver.find_element(By.ID, "moreSongs")
|
||||
more_button.click()
|
||||
time.sleep(5) # Wait for more songs to load
|
||||
except:
|
||||
pass # Button not found or already loaded
|
||||
|
||||
html = driver.page_source
|
||||
driver.quit()
|
||||
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
songs = []
|
||||
# Find elements (div, li) that contain time in their text
|
||||
song_elements = soup.find_all(lambda tag: tag.name in ['div', 'li', 'p'] and re.search(r'\d+:\d+ [AP]M', tag.get_text()))
|
||||
for element in song_elements:
|
||||
spans = element.find_all('span')
|
||||
if len(spans) >= 3:
|
||||
title = spans[0].get_text().strip()
|
||||
artist = spans[1].get_text().strip()
|
||||
# Ignore time
|
||||
songs.append({'artist': artist, 'title': title})
|
||||
else:
|
||||
# Fallback to text parsing
|
||||
full_text = element.get_text().strip()
|
||||
match = re.search(r'(\d+:\d+ [AP]M)', full_text)
|
||||
if match:
|
||||
play_time = match.group(1)
|
||||
before = full_text.split(play_time)[0].strip()
|
||||
if ' by ' in before:
|
||||
title, artist = before.split(' by ', 1)
|
||||
title = title.strip()
|
||||
artist = artist.strip()
|
||||
else:
|
||||
title = ' '.join(parts[:3])
|
||||
artist = ' '.join(parts[3:])
|
||||
if title and artist:
|
||||
songs.append({'artist': artist, 'title': title})
|
||||
# Split on capital letters
|
||||
parts = re.findall(r'[A-Z][^A-Z]*', before)
|
||||
if len(parts) == 4:
|
||||
title = ' '.join(parts[:2])
|
||||
artist = ' '.join(parts[2:])
|
||||
else:
|
||||
title = ' '.join(parts[:3])
|
||||
artist = ' '.join(parts[3:])
|
||||
if title and artist:
|
||||
songs.append({'artist': artist, 'title': title})
|
||||
|
||||
# Add to all_songs, avoiding duplicates
|
||||
for song in songs:
|
||||
key = (song['title'].lower(), song['artist'].lower())
|
||||
if key not in seen:
|
||||
all_songs.append(song)
|
||||
seen.add(key)
|
||||
|
||||
return songs
|
||||
return all_songs
|
||||
|
||||
def normalize_string(s):
|
||||
"""Normalize string for comparison"""
|
||||
|
||||
Reference in new issue
Block a user