Add scraping from Spirit FM recently played list, combine with Journey FM and deduplicate songs
This commit is contained in:
1 parent
7e7754f533
commit
32d87cf0c2
1 file changed
+18
-7
@@ -31,7 +31,16 @@ SERVER_IP = '172.16.16.106' # Your local server IP
|
|||||||
PLAYLIST_NAME = 'Journey FM Recently Played'
|
PLAYLIST_NAME = 'Journey FM Recently Played'
|
||||||
|
|
||||||
def scrape_recently_played():
|
def scrape_recently_played():
|
||||||
"""Scrape recently played songs from myjourneyfm.com using Selenium"""
|
"""Scrape recently played songs from multiple stations using Selenium"""
|
||||||
|
urls = [
|
||||||
|
'https://www.myjourneyfm.com/recently-played/',
|
||||||
|
'https://spiritfm.com/spiritfm-recently-played/'
|
||||||
|
]
|
||||||
|
|
||||||
|
all_songs = []
|
||||||
|
seen = set() # To avoid duplicates
|
||||||
|
|
||||||
|
for url in urls:
|
||||||
options = Options()
|
options = Options()
|
||||||
options.add_argument("--headless")
|
options.add_argument("--headless")
|
||||||
options.add_argument("--no-sandbox")
|
options.add_argument("--no-sandbox")
|
||||||
@@ -45,7 +54,6 @@ def scrape_recently_played():
|
|||||||
service = Service()
|
service = Service()
|
||||||
driver = webdriver.Chrome(service=service, options=options)
|
driver = webdriver.Chrome(service=service, options=options)
|
||||||
|
|
||||||
url = 'https://www.myjourneyfm.com/recently-played/'
|
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
|
|
||||||
# Wait for initial load
|
# Wait for initial load
|
||||||
@@ -62,10 +70,6 @@ def scrape_recently_played():
|
|||||||
html = driver.page_source
|
html = driver.page_source
|
||||||
driver.quit()
|
driver.quit()
|
||||||
|
|
||||||
# Optional: save HTML to file for debugging (comment out if not needed)
|
|
||||||
# with open('page.html', 'w', encoding='utf-8') as f:
|
|
||||||
# f.write(html)
|
|
||||||
|
|
||||||
soup = BeautifulSoup(html, 'html.parser')
|
soup = BeautifulSoup(html, 'html.parser')
|
||||||
|
|
||||||
songs = []
|
songs = []
|
||||||
@@ -101,7 +105,14 @@ def scrape_recently_played():
|
|||||||
if title and artist:
|
if title and artist:
|
||||||
songs.append({'artist': artist, 'title': title})
|
songs.append({'artist': artist, 'title': title})
|
||||||
|
|
||||||
return songs
|
# Add to all_songs, avoiding duplicates
|
||||||
|
for song in songs:
|
||||||
|
key = (song['title'].lower(), song['artist'].lower())
|
||||||
|
if key not in seen:
|
||||||
|
all_songs.append(song)
|
||||||
|
seen.add(key)
|
||||||
|
|
||||||
|
return all_songs
|
||||||
|
|
||||||
def normalize_string(s):
|
def normalize_string(s):
|
||||||
"""Normalize string for comparison"""
|
"""Normalize string for comparison"""
|
||||||
|
|||||||
Reference in new issue
Block a user