From e762c4252de7e22146805b4543ac155391feaa60 Mon Sep 17 00:00:00 2001 From: nmemmert Date: Mon, 17 Nov 2025 13:41:25 -0500 Subject: [PATCH] Fix song scraping - use proper HTML structure parsing --- amazon_buy_list.txt | 21 ++++++ app_log.txt | 12 ++++ main.py | 172 +++++++++++++++++++++++--------------------- playlist_history.db | Bin 16384 -> 16384 bytes 4 files changed, 122 insertions(+), 83 deletions(-) diff --git a/amazon_buy_list.txt b/amazon_buy_list.txt index 85be51c..70eb17b 100644 --- a/amazon_buy_list.txt +++ b/amazon_buy_list.txt @@ -141,3 +141,24 @@ https://www.amazon.com/s?k=You%20Danny%20%20Gokey%20I%20%20Got%20&i=digital-musi You've Done Tasha Layton - Look What https://www.amazon.com/s?k=You%27ve%20%20Done%20Tasha%20%20Layton%20Look%20%20What%20&i=digital-music +Zach Williams - Jesus Loves +https://www.amazon.com/s?k=Zach%20Williams%20Jesus%20Loves&i=digital-music + +Zach Williams - Jesus Loves +https://www.amazon.com/s?k=Zach%20%20Williams%20Jesus%20%20Loves&i=digital-music + +Am They - Scars I +https://www.amazon.com/s?k=Am%20%20They%20Scars%20I%20&i=digital-music + +Church Sing Tauren Wells - Let The +https://www.amazon.com/s?k=Church%20%20Sing%20Tauren%20%20Wells%20Let%20%20The%20&i=digital-music + +You Danny Gokey - I Got +https://www.amazon.com/s?k=You%20Danny%20%20Gokey%20I%20%20Got%20&i=digital-music + +Cochren & Co. - Church (Take Me Back) +https://www.amazon.com/s?k=Cochren%20%26%20Co.%20Church%20%28Take%20Me%20Back%29&i=digital-music + +Blessing Offor - Your Love +https://www.amazon.com/s?k=Blessing%20Offor%20Your%20Love&i=digital-music + diff --git a/app_log.txt b/app_log.txt index 37b2d83..105b495 100644 --- a/app_log.txt +++ b/app_log.txt @@ -4,3 +4,15 @@ 2025-11-17 12:42:52.626588: Starting Journey FM Playlist app... 2025-11-17 12:42:52.626688: DISPLAY set, initializing GUI 2025-11-17 12:42:52.761510: App initialized, starting event loop +2025-11-17 12:56:44.139926: Starting Journey FM Playlist app... +2025-11-17 12:56:44.140042: DISPLAY set, initializing GUI +2025-11-17 12:56:44.381412: App initialized, starting event loop +2025-11-17 13:01:57.291192: Starting Journey FM Playlist app... +2025-11-17 13:01:57.291370: DISPLAY set, initializing GUI +2025-11-17 13:01:57.440291: App initialized, starting event loop +2025-11-17 13:09:29.113472: Starting Journey FM Playlist app... +2025-11-17 13:09:29.113563: DISPLAY set, initializing GUI +2025-11-17 13:09:29.245236: App initialized, starting event loop +2025-11-17 13:12:14.103261: Starting Journey FM Playlist app... +2025-11-17 13:12:14.103373: DISPLAY set, initializing GUI +2025-11-17 13:12:14.295725: App initialized, starting event loop diff --git a/main.py b/main.py index bc528d9..be68a56 100644 --- a/main.py +++ b/main.py @@ -13,6 +13,8 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC from bs4 import BeautifulSoup import os from plexapi.myplex import MyPlexAccount @@ -31,103 +33,107 @@ SERVER_IP = '172.16.16.106' # Your local server IP PLAYLIST_NAME = 'Journey FM Recently Played' def scrape_recently_played(): - """Scrape recently played songs from multiple stations using Selenium""" - import platform + """Scrape recently played songs from Journey FM and Spirit FM""" + options = Options() + options.add_argument("--headless=new") + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-gpu") + options.add_argument("--window-size=1920,1080") + options.binary_location = "/usr/bin/google-chrome-stable" - urls = [ - ('https://www.myjourneyfm.com/recently-played/', 'Journey FM'), - ('https://spiritfm.com/spiritfm-recently-played/', 'Spirit FM') - ] + service = Service(ChromeDriverManager().install()) + + try: + driver = webdriver.Chrome(service=service, options=options) + except Exception as e: + print(f"Failed to start Chrome: {e}") + return [] all_songs = [] - seen = set() # To avoid duplicates + seen = set() - for url, source in urls: - options = Options() - options.add_argument("--headless") - options.add_argument("--no-sandbox") - options.add_argument("--disable-dev-shm-usage") - options.add_argument("--disable-gpu") - - service = Service(ChromeDriverManager().install()) - driver = webdriver.Chrome(service=service, options=options) - - driver.get(url) - - # Wait for initial load - time.sleep(10) + # Journey FM - uses for title, normal text for artist + try: + print("Scraping Journey FM...") + driver.get('https://www.myjourneyfm.com/recently-played/') + time.sleep(5) # Click "View More" button if exists try: more_button = driver.find_element(By.ID, "moreSongs") more_button.click() - time.sleep(10) # Wait for more songs to load + time.sleep(5) except: - pass # Button not found or already loaded + pass - try: - html = driver.page_source - except Exception as e: - print(f"Failed to get page source for {source}: {e}") - html = "" + soup = BeautifulSoup(driver.page_source, 'html.parser') - driver.quit() + # Find all song items using the proper class structure + song_items = soup.find_all('div', class_='rp-item') - soup = BeautifulSoup(html, 'html.parser') - - songs = [] - # Find elements (div, li) that contain time in their text - song_elements = soup.find_all(lambda tag: tag.name in ['div', 'li', 'p'] and re.search(r'\d+:\d+', tag.get_text())) - for element in song_elements: - spans = element.find_all('span') - if len(spans) >= 2: - title = spans[0].get_text().strip() - artist = spans[1].get_text().strip() - else: - # Check for strong - strong = element.find('strong') - if strong: - title = strong.get_text().strip() - full_text = element.get_text().strip() - match = re.search(r'(\d+:\d+(?: [AP]M)?)', full_text) - if match: - time_part = match.group(1) - before_time = full_text.split(time_part)[0].strip() - artist = before_time.replace(title, '').strip() - else: - artist = full_text.replace(title, '').strip() - artist = artist.strip() - else: - # Fallback to text parsing - full_text = element.get_text().strip() - match = re.search(r'(\d+:\d+(?: [AP]M)?)', full_text) - if match: - play_time = match.group(1) - before = full_text.split(play_time)[0].strip() - if ' by ' in before: - title, artist = before.split(' by ', 1) - title = title.strip() - artist = artist.strip() - elif ' - ' in before: - title, artist = before.split(' - ', 1) - title = title.strip() - artist = artist.strip() - else: - parts = re.findall(r'[A-Z][^A-Z]*', before) - if len(parts) >= 4: - title = ' '.join(parts[:2]) - artist = ' '.join(parts[2:]) - else: - title = ' '.join(parts[:len(parts)//2]) - artist = ' '.join(parts[len(parts)//2:]) - if title and artist: - songs.append({'artist': artist, 'title': title, 'source': source}) # Add to all_songs, avoiding duplicates - for song in songs: - key = (song['title'].lower(), song['artist'].lower()) - if key not in seen: - all_songs.append(song) - seen.add(key) + for item in song_items: + try: + title_elem = item.find('h5', class_='song-title') + artist_elem = item.find('p', class_='song-artist') + + if title_elem and artist_elem: + title = title_elem.get_text().strip() + artist = artist_elem.get_text().strip() + + if title and artist: + key = (title.lower(), artist.lower()) + if key not in seen: + all_songs.append({'title': title, 'artist': artist, 'source': 'Journey FM'}) + seen.add(key) + print(f" Found: {title} by {artist}") + except Exception as e: + continue + + except Exception as e: + print(f"Error scraping Journey FM: {e}") + # Spirit FM - uses Title - Artist format + try: + print("Scraping Spirit FM...") + driver.get('https://spiritfm.com/spiritfm-recently-played/') + time.sleep(5) + + soup = BeautifulSoup(driver.page_source, 'html.parser') + + # Find all playlist items + items = soup.find_all(lambda tag: tag.name in ['div', 'li'] and re.search(r'\d+:\d+\s+[AP]M', tag.get_text())) + + for item in items: + try: + text = item.get_text().strip() + + # Remove time + time_match = re.search(r'\d+:\d+\s+[AP]M', text) + if time_match: + before_time = text.split(time_match.group())[0].strip() + + # Spirit FM uses dash separator + if ' - ' in before_time: + parts = before_time.split(' - ', 1) + title = parts[0].strip() + artist = parts[1].strip() + + if title and artist: + key = (title.lower(), artist.lower()) + if key not in seen: + all_songs.append({'title': title, 'artist': artist, 'source': 'Spirit FM'}) + seen.add(key) + print(f" Found: {title} by {artist}") + except Exception as e: + continue + + except Exception as e: + print(f"Error scraping Spirit FM: {e}") + + driver.quit() + + print(f"\nTotal unique songs found: {len(all_songs)}") return all_songs def normalize_string(s): diff --git a/playlist_history.db b/playlist_history.db index 7ec095e6c73c4c583f2cb965cbbd0080567cb62f..c84c4451e40fcc4a7a3d6b80b222991ce976c295 100644 GIT binary patch delta 1203 zcmc(eUu)A)7>Ac`aaCbjbxy5F3}&vlou0orCr1&sf(%i-a3C%O3|(f!x`idFHzMse z&>KNZzk**t`bl={r|=tSny_djyPfyX@9=xy^FD)tG8iad>q>oiyQS2>zxmc&tSODD zxeFH`p8q&|to~5P>SxtcAFB&9KWB{T&B~Wbr?RRvhWo#^4(D%Gr&g;q07k>RcR-$g9rQx&S@6y)Cf2%M6#08^wHMplK{$yt^T<;#>P2ZLDuy=Q zq{l12+x6YuXtcjv?;AH~C*xFW#c7C;)C8Cd6GPjv9pgqh#wB6^nj9f>f*q_cWZOA> z`OkYyG6FbcgmT-dO!RlTGwn8iN;e`Q16<%_$^WNY z-?&!(_efF!EGjrl|K7M-jzLMx#0!XoNH6J1DTbM(7!X1!Bq>%q!TJTwFp-1-f-Oq3 z=PsXcMGQ^ujUvje47*f{AubsPf)mS1q_v%+crN>fPY?3@eUj*nCF7D3U}BT<jgY%0x^IUAmu=xqu~TChZ74YK%YV_Y}n$AMH=0|4=!3@HEr