nate/journey-fm-playlist
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add scraping from Spirit FM recently played list, combine with Journey FM and deduplicate songs

Browse Source
This commit is contained in:
nmemmert committed 2025-11-17 11:56:26 -05:00
1 parent 7e7754f533
commit 32d87cf0c2
1 file changed
+73 -62
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+73 -62
View File
@@ -31,77 +31,88 @@ SERVER_IP = '172.16.16.106' # Your local server IP
PLAYLIST_NAME = 'Journey FM Recently Played' PLAYLIST_NAME = 'Journey FM Recently Played'
def scrape_recently_played(): def scrape_recently_played():
"""Scrape recently played songs from myjourneyfm.com using Selenium""" """Scrape recently played songs from multiple stations using Selenium"""
options = Options() urls = [
options.add_argument("--headless") 'https://www.myjourneyfm.com/recently-played/',
options.add_argument("--no-sandbox") 'https://spiritfm.com/spiritfm-recently-played/'
options.add_argument("--disable-dev-shm-usage") ]
options.add_argument("--disable-gpu")
options.add_argument("--remote-debugging-port=9222")
options.add_argument("--user-data-dir=/tmp/chromium")
options.add_argument("--no-first-run")
options.add_argument("--disable-extensions")
options.binary_location = "/usr/bin/chromium-browser"
service = Service()
driver = webdriver.Chrome(service=service, options=options)
url = 'https://www.myjourneyfm.com/recently-played/' all_songs = []
driver.get(url) seen = set() # To avoid duplicates
# Wait for initial load for url in urls:
time.sleep(5) options = Options()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-gpu")
options.add_argument("--remote-debugging-port=9222")
options.add_argument("--user-data-dir=/tmp/chromium")
options.add_argument("--no-first-run")
options.add_argument("--disable-extensions")
options.binary_location = "/usr/bin/chromium-browser"
service = Service()
driver = webdriver.Chrome(service=service, options=options)
# Click "View More" button if exists driver.get(url)
try:
more_button = driver.find_element(By.ID, "moreSongs")
more_button.click()
time.sleep(5) # Wait for more songs to load
except:
pass # Button not found or already loaded
html = driver.page_source # Wait for initial load
driver.quit() time.sleep(5)
# Optional: save HTML to file for debugging (comment out if not needed) # Click "View More" button if exists
# with open('page.html', 'w', encoding='utf-8') as f: try:
# f.write(html) more_button = driver.find_element(By.ID, "moreSongs")
more_button.click()
time.sleep(5) # Wait for more songs to load
except:
pass # Button not found or already loaded
soup = BeautifulSoup(html, 'html.parser') html = driver.page_source
driver.quit()
songs = [] soup = BeautifulSoup(html, 'html.parser')
# Find elements (div, li) that contain time in their text
song_elements = soup.find_all(lambda tag: tag.name in ['div', 'li', 'p'] and re.search(r'\d+:\d+ [AP]M', tag.get_text())) songs = []
for element in song_elements: # Find elements (div, li) that contain time in their text
spans = element.find_all('span') song_elements = soup.find_all(lambda tag: tag.name in ['div', 'li', 'p'] and re.search(r'\d+:\d+ [AP]M', tag.get_text()))
if len(spans) >= 3: for element in song_elements:
title = spans[0].get_text().strip() spans = element.find_all('span')
artist = spans[1].get_text().strip() if len(spans) >= 3:
# Ignore time title = spans[0].get_text().strip()
songs.append({'artist': artist, 'title': title}) artist = spans[1].get_text().strip()
else: # Ignore time
# Fallback to text parsing songs.append({'artist': artist, 'title': title})
full_text = element.get_text().strip() else:
match = re.search(r'(\d+:\d+ [AP]M)', full_text) # Fallback to text parsing
if match: full_text = element.get_text().strip()
play_time = match.group(1) match = re.search(r'(\d+:\d+ [AP]M)', full_text)
before = full_text.split(play_time)[0].strip() if match:
if ' by ' in before: play_time = match.group(1)
title, artist = before.split(' by ', 1) before = full_text.split(play_time)[0].strip()
title = title.strip() if ' by ' in before:
artist = artist.strip() title, artist = before.split(' by ', 1)
else: title = title.strip()
# Split on capital letters artist = artist.strip()
parts = re.findall(r'[A-Z][^A-Z]*', before)
if len(parts) == 4:
title = ' '.join(parts[:2])
artist = ' '.join(parts[2:])
else: else:
title = ' '.join(parts[:3]) # Split on capital letters
artist = ' '.join(parts[3:]) parts = re.findall(r'[A-Z][^A-Z]*', before)
if title and artist: if len(parts) == 4:
songs.append({'artist': artist, 'title': title}) title = ' '.join(parts[:2])
artist = ' '.join(parts[2:])
else:
title = ' '.join(parts[:3])
artist = ' '.join(parts[3:])
if title and artist:
songs.append({'artist': artist, 'title': title})
return songs # Add to all_songs, avoiding duplicates
for song in songs:
key = (song['title'].lower(), song['artist'].lower())
if key not in seen:
all_songs.append(song)
seen.add(key)
return all_songs
def normalize_string(s): def normalize_string(s):
"""Normalize string for comparison""" """Normalize string for comparison"""