From da2b725b227297e4baa9a8a749f28e05f0b33ef4 Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 26 Nov 2024 15:49:40 +0000 Subject: [PATCH 01/42] updated github workflow no2 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eef75d7..f3e41e4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,7 +59,7 @@ jobs: - name: Generate Auto Release Notes id: release_notes run: | - auto_notes=$(gh release create ${{ env.VERSION }} --generate-notes --prerelease --dry-run --json body --jq .body) + auto_notes=$(gh release create ${{ env.VERSION }} --generate-notes --prerelease --json body --jq .body) echo "AUTO_RELEASE_NOTES<> $GITHUB_ENV echo "$auto_notes" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV From ddf73b77db7cbfa52687c3f8f1416d1bba191802 Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 27 Nov 2024 16:03:39 +0000 Subject: [PATCH 02/42] provide more technical track details in the ui Fixes #15 --- app/__init__.py | 8 ++- app/classes.py | 79 ++++++++++++++++++++++ app/filters.py | 48 +++++++++++++ app/jellyfin_routes.py | 2 +- config.py | 1 + static/css/styles.css | 5 ++ templates/partials/_jf_search_results.html | 14 +++- templates/partials/_track_table.html | 2 +- 8 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 app/classes.py create mode 100644 app/filters.py diff --git a/app/__init__.py b/app/__init__.py index 110a309..33a62bc 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -163,4 +163,10 @@ app.logger.debug(f"Debug logging active") from app import routes from app import jellyfin_routes, tasks if "worker" in sys.argv: - tasks.release_lock("download_missing_tracks_lock") \ No newline at end of file + tasks.release_lock("download_missing_tracks_lock") + +from app import filters # Import the filters dictionary + +# Register all filters +for name, func in filters.filters.items(): + app.jinja_env.filters[name] = func \ No newline at end of file diff --git a/app/classes.py b/app/classes.py new file mode 100644 index 0000000..56a9bec --- /dev/null +++ b/app/classes.py @@ -0,0 +1,79 @@ +import subprocess +import json +from flask import current_app as app # Adjust this based on your app's structure +from typing import Optional + + +class AudioProfile: + def __init__(self, path: str, bitrate: int = 0, sample_rate: int = 0, channels: int = 0) -> None: + """ + Initialize an AudioProfile instance. + + Args: + path (str): The file path of the audio file. + bitrate (int): The audio bitrate in kbps. Default is 0. + sample_rate (int): The sample rate in Hz. Default is 0. + channels (int): The number of audio channels. Default is 0. + """ + self.path: str = path + self.bitrate: int = bitrate # in kbps + self.sample_rate: int = sample_rate # in Hz + self.channels: int = channels + + @staticmethod + def analyze_audio_quality_with_ffprobe(filepath: str) -> Optional['AudioProfile']: + """ + Static method to analyze audio quality using ffprobe and return an AudioProfile instance. + + Args: + filepath (str): Path to the audio file to analyze. + + Returns: + Optional[AudioProfile]: An instance of AudioProfile if analysis is successful, None otherwise. + """ + try: + # ffprobe command to extract bitrate, sample rate, and channel count + cmd = [ + 'ffprobe', '-v', 'error', '-select_streams', 'a:0', + '-show_entries', 'stream=bit_rate,sample_rate,channels', + '-show_format', + '-of', 'json', filepath + ] + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode != 0: + app.logger.error(f"ffprobe error for file {filepath}: {result.stderr}") + return None + + # Parse ffprobe output + data = json.loads(result.stdout) + stream = data.get('streams', [{}])[0] + bitrate: int = int(stream.get('bit_rate', 0)) // 1000 # Convert to kbps + if bitrate == 0: # Fallback if no bit_rate in stream + bitrate = int(data.get('format').get('bit_rate', 0)) // 1000 + sample_rate: int = int(stream.get('sample_rate', 0)) # Hz + channels: int = int(stream.get('channels', 0)) + + # Create an AudioProfile instance + return AudioProfile(filepath, bitrate, sample_rate, channels) + except Exception as e: + app.logger.error(f"Error analyzing audio quality with ffprobe: {str(e)}") + return None + + def compute_quality_score(self) -> int: + """ + Compute a quality score based on bitrate, sample rate, and channels. + + Returns: + int: The computed quality score. + """ + return self.bitrate + (self.sample_rate // 1000) + (self.channels * 10) + + def __repr__(self) -> str: + """ + Representation of the AudioProfile instance. + + Returns: + str: A string representation of the AudioProfile instance. + """ + return (f"AudioProfile(path='{self.path}', bitrate={self.bitrate} kbps, " + f"sample_rate={self.sample_rate} Hz, channels={self.channels})") diff --git a/app/filters.py b/app/filters.py new file mode 100644 index 0000000..1bc4ec7 --- /dev/null +++ b/app/filters.py @@ -0,0 +1,48 @@ +import os +import re +from markupsafe import Markup + +from app.classes import AudioProfile + +filters = {} + +def template_filter(name): + """Decorator to register a Jinja2 filter.""" + def decorator(func): + filters[name] = func + return func + return decorator + +@template_filter('highlight') +def highlight_search(text, search_query): + if not search_query: + return text + search_query_escaped = re.escape(search_query) + highlighted_text = re.sub( + f"({search_query_escaped})", + r'\1', + text, + flags=re.IGNORECASE + ) + return Markup(highlighted_text) + + +@template_filter('audioprofile') +def audioprofile(text: str, path: str) -> Markup: + if not path or not os.path.exists(path): + return Markup() # Return the original text if the file does not exist + + # Create the AudioProfile instance using the static method + audio_profile = AudioProfile.analyze_audio_quality_with_ffprobe(path) + if not audio_profile: + return Markup(f"ERROR") + + # Create a nicely formatted HTML representation + audio_profile_html = ( + f"Bitrate: {audio_profile.bitrate} kbps
" + f"Sample Rate: {audio_profile.sample_rate} Hz
" + f"Channels: {audio_profile.channels}
" + f"Quality Score: {audio_profile.compute_quality_score()}" + f"" + ) + return Markup(audio_profile_html) diff --git a/app/jellyfin_routes.py b/app/jellyfin_routes.py index 1f99f39..90978d0 100644 --- a/app/jellyfin_routes.py +++ b/app/jellyfin_routes.py @@ -208,5 +208,5 @@ def search_jellyfin(): if search_query: results = jellyfin.search_music_tracks(functions._get_token_from_sessioncookie(), search_query) # Render only the search results section as response - return render_template('partials/_jf_search_results.html', results=results,spotify_id= spotify_id) + return render_template('partials/_jf_search_results.html', results=results,spotify_id= spotify_id,search_query = search_query) return jsonify({'error': 'No search query provided'}), 400 \ No newline at end of file diff --git a/config.py b/config.py index fc0436d..3fc6dae 100644 --- a/config.py +++ b/config.py @@ -17,6 +17,7 @@ class Config: JELLYPLIST_DB_PASSWORD = os.getenv('JELLYPLIST_DB_PASSWORD') START_DOWNLOAD_AFTER_PLAYLIST_ADD = os.getenv('START_DOWNLOAD_AFTER_PLAYLIST_ADD',"false").lower() == 'true' # If a new Playlist is added, the Download Task will be scheduled immediately REFRESH_LIBRARIES_AFTER_DOWNLOAD_TASK = os.getenv('REFRESH_LIBRARIES_AFTER_DOWNLOAD_TASK',"false").lower() == 'true' + DISPLAY_EXTENDED_AUDIO_DATA = os.getenv('DISPLAY_EXTENDED_AUDIO_DATA',"false").lower() == 'true' CACHE_TYPE = 'redis' CACHE_REDIS_PORT = 6379 CACHE_REDIS_HOST = 'redis' diff --git a/static/css/styles.css b/static/css/styles.css index 2ef654b..faedd5a 100644 --- a/static/css/styles.css +++ b/static/css/styles.css @@ -52,4 +52,9 @@ body { } .logo img{ width: 100%; +} +@media screen and (min-width: 1600px) { + .modal-dialog { + max-width: 90%; /* New width for default modal */ + } } \ No newline at end of file diff --git a/templates/partials/_jf_search_results.html b/templates/partials/_jf_search_results.html index c50346f..b517eef 100644 --- a/templates/partials/_jf_search_results.html +++ b/templates/partials/_jf_search_results.html @@ -7,6 +7,10 @@ Name Artist(s) Path + Container + {% if config['DISPLAY_EXTENDED_AUDIO_DATA'] %} + + {% endif %} @@ -14,17 +18,21 @@ {% for track in results %} - {{ track.Name }} + {{ track.Name | highlight(search_query) }} {{ ', '.join(track.Artists) }} {{ track.Path}} + {{ track.Container }} + {% if config['DISPLAY_EXTENDED_AUDIO_DATA'] %} + {{track.Path | audioprofile(track.Path) }} + {% endif %} - diff --git a/templates/partials/_track_table.html b/templates/partials/_track_table.html index eec4663..972e2b4 100644 --- a/templates/partials/_track_table.html +++ b/templates/partials/_track_table.html @@ -71,7 +71,7 @@ + + From 18dc6e18afd327f630728518563b7bef8c111738 Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 27 Nov 2024 20:07:13 +0000 Subject: [PATCH 09/42] rework on find_best_match_from_jellyfin - Artists will be compared, even if only one search result from jellyfin #24 --- app/tasks.py | 62 ++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/app/tasks.py b/app/tasks.py index 60e0646..815b399 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -400,7 +400,7 @@ def update_jellyfin_id_for_downloaded_tracks(self): processed_tracks += 1 progress = (processed_tracks / total_tracks) * 100 - self.update_state(state='PROGRESS', meta={'current': processed_tracks, 'total': total_tracks, 'percent': progress}) + self.update_state(state=f'{processed_tracks}/{total_tracks}: {track.name}', meta={'current': processed_tracks, 'total': total_tracks, 'percent': progress}) app.logger.info("Finished updating Jellyfin IDs for all tracks.") return {'status': 'All tracks updated', 'total': total_tracks, 'processed': processed_tracks} @@ -421,37 +421,43 @@ def find_best_match_from_jellyfin(track: Track): for result in search_results: + + app.logger.debug(f"Processing search result: {result['Id']}") quality_score = compute_quality_score(result, app.config['FIND_BEST_MATCH_USE_FFPROBE']) - - if len(search_results) == 1: - app.logger.debug(f"Only 1 search_result, assuming best match: {result['Id']} ({app.config['JELLYFIN_SERVER_URL']}/web/#/details?id={result['Id']})") - best_match = result - break - - jellyfin_path = result.get('Path', '') - # if jellyfin_path == track.filesystem_path: - # app.logger.debug(f"Best match found through equal file-system paths: {result['Id']} ({app.config['JELLYFIN_SERVER_URL']}/web/#/details?id={result['Id']})") - # best_match = result - # break - - if not spotify_track: - try: - spotify_track = functions.get_cached_spotify_track(track.spotify_track_id) - spotify_track_name = spotify_track['name'] - spotify_artists = [artist['name'] for artist in spotify_track['artists']] - except Exception as e: - app.logger.error(f"Error fetching track details from Spotify for {track.name}: {str(e)}") - continue - + try: + spotify_track = functions.get_cached_spotify_track(track.spotify_track_id) + spotify_track_name = spotify_track['name'].lower() + spotify_artists = [artist['name'].lower() for artist in spotify_track['artists']] + except Exception as e: + app.logger.error(f"\tError fetching track details from Spotify for {track.name}: {str(e)}") + continue jellyfin_track_name = result.get('Name', '').lower() jellyfin_artists = [artist.lower() for artist in result.get('Artists', [])] - if (spotify_track_name.lower() == jellyfin_track_name and - set(artist.lower() for artist in spotify_artists) == set(jellyfin_artists)): - app.logger.debug(f"Quality score for track {result['Name']}: {quality_score} [{result['Path']}]") + + if spotify_track and jellyfin_track_name and jellyfin_artists and spotify_artists: + app.logger.debug("\tTrack details to compare: ") + app.logger.debug(f"\t\tJellyfin-Trackname : {jellyfin_track_name}") + app.logger.debug(f"\t\t Spotify-Trackname : {spotify_track_name}") + app.logger.debug(f"\t\t Jellyfin-Artists : {jellyfin_artists}") + app.logger.debug(f"\t\t Spotify-Artists : {spotify_artists}") + if len(search_results) == 1: + app.logger.debug(f"\tOnly 1 search_result: {result['Id']} ({app.config['JELLYFIN_SERVER_URL']}/web/#/details?id={result['Id']})") + + if (spotify_track_name.lower() == jellyfin_track_name and + set(artist.lower() for artist in spotify_artists) == set(jellyfin_artists)): + + app.logger.debug(f"\tQuality score for track {result['Name']}: {quality_score} [{result['Path']}]") + best_match = result + break - if quality_score > best_quality_score: - best_match = result - best_quality_score = quality_score + + if (spotify_track_name.lower() == jellyfin_track_name and + set(artist.lower() for artist in spotify_artists) == set(jellyfin_artists)): + app.logger.debug(f"\tQuality score for track {result['Name']}: {quality_score} [{result['Path']}]") + + if quality_score > best_quality_score: + best_match = result + best_quality_score = quality_score return best_match except Exception as e: From d8d677bc1b5190e3bbc40018bfd874f27dde1761 Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 27 Nov 2024 20:08:03 +0000 Subject: [PATCH 10/42] UI Fix on Task overview --- templates/admin/tasks.html | 7 +++++-- templates/partials/_task_status.html | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/templates/admin/tasks.html b/templates/admin/tasks.html index ef6670e..d0296be 100644 --- a/templates/admin/tasks.html +++ b/templates/admin/tasks.html @@ -1,7 +1,7 @@ {% extends "admin.html" %} {% block admin_content %} -
- +
+
@@ -15,6 +15,9 @@
Task Name
+ +
+

Unlock blocked tasks

diff --git a/templates/partials/_task_status.html b/templates/partials/_task_status.html index c32f89c..53a02ad 100644 --- a/templates/partials/_task_status.html +++ b/templates/partials/_task_status.html @@ -1,7 +1,7 @@ {% for task_name, task in tasks.items() %} - {{ task_name }} - {{ task.state }} + {{ task_name }} + {{ task.state }} {% if task.info.percent %}
From cbe172ff1f340b6a04be15548dcb50f6a4e30fb0 Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 19:55:07 +0000 Subject: [PATCH 11/42] base classes for generic musicProviderClient --- app/providers/base.py | 131 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 app/providers/base.py diff --git a/app/providers/base.py b/app/providers/base.py new file mode 100644 index 0000000..a49d322 --- /dev/null +++ b/app/providers/base.py @@ -0,0 +1,131 @@ +from dataclasses import dataclass, field +from typing import List, Optional +from abc import ABC, abstractmethod + +@dataclass +class ExternalUrl: + url: str + +@dataclass +class ItemBase: + id: str + name: str + uri: str + external_urls: Optional[List[ExternalUrl]] + +@dataclass +class Image: + url: str + height: Optional[int] + width: Optional[int] + +@dataclass +class Artist(ItemBase): + pass + +@dataclass +class Album(ItemBase): + artists: List[Artist] + images: List[Image] + +@dataclass +class Track(ItemBase): + duration_ms: int + explicit: Optional[bool] + album: Optional[Album] + artists: List[Artist] + +@dataclass +class PlaylistTrack: + added_at: Optional[str] + added_by: Optional[str] + is_local: bool + track: Track + +@dataclass +class Owner(ItemBase): + pass + +@dataclass #tbc +class Category(ItemBase): + pass + +@dataclass +class Playlist(ItemBase): + + description: Optional[str] + public: Optional[bool] + collaborative: Optional[bool] + followers: Optional[int] + images: Optional[List[Image]] + owner: Optional[Owner] + tracks: List[PlaylistTrack] = field(default_factory=list) + +# Abstract base class for music providers +class MusicProviderClient(ABC): + """ + Abstract base class defining the interface for music provider clients. + """ + + @abstractmethod + def authenticate(self, credentials: dict) -> None: + """ + Authenticates the client with the provider using the provided credentials. + :param credentials: A dictionary containing credentials (e.g., API keys, tokens). + """ + pass + + @abstractmethod + def get_playlist(self, playlist_id: str) -> Playlist: + """ + Fetches a playlist by its ID. + :param playlist_id: The ID of the playlist to fetch. + :return: A Playlist object. + """ + pass + + @abstractmethod + def search_tracks(self, query: str, limit: int = 50) -> List[Track]: + """ + Searches for tracks based on a query string. + :param query: The search query. + :param limit: Maximum number of results to return. + :return: A list of Track objects. + """ + pass + + @abstractmethod + def get_track(self, track_id: str) -> Track: + """ + Fetches details for a specific track. + :param track_id: The ID of the track to fetch. + :return: A Track object. + """ + pass + @abstractmethod + def get_featured_playlists(self, limit: int = 50) -> List[Playlist]: + """ + Fetches a list of featured playlists. + :param limit: Maximum number of featured playlists to return. + :return: A list of Playlist objects. + """ + pass + + @abstractmethod + def get_playlists_by_category(self, category_id: str, limit: int = 50) -> List[Playlist]: + """ + Fetches playlists belonging to a specific category. + :param category_id: The ID of the category. + :param limit: Maximum number of playlists to return. + :return: A list of Playlist objects. + """ + pass + + @abstractmethod + def get_categories(self, limit: int = 50) -> List[Category]: + """ + Fetches a list of available categories. + :param limit: Maximum number of categories to return. + :return: A list of categories, where each category is a dictionary with 'id' and 'name'. + """ + pass \ No newline at end of file From f81188f7e3b22bb188727fed239cce37dab2dc82 Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 19:55:27 +0000 Subject: [PATCH 12/42] spotify client using generic base classes --- app/providers/spotify.py | 207 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 app/providers/spotify.py diff --git a/app/providers/spotify.py b/app/providers/spotify.py new file mode 100644 index 0000000..6fcf145 --- /dev/null +++ b/app/providers/spotify.py @@ -0,0 +1,207 @@ +import os +from app.providers.base import Album, MusicProviderClient,Playlist,Track,ExternalUrl,Category +import requests + +import json +import requests +from bs4 import BeautifulSoup +from urllib.parse import urlencode +from typing import List, Dict, Optional +from http.cookiejar import MozillaCookieJar + +class SpotifyClient(MusicProviderClient): + """ + Spotify implementation of the MusicProviderClient. + """ + + def __init__(self, cookie_file: Optional[str] = None): + self.base_url = "https://api-partner.spotify.com" + self.session_data = None + self.config_data = None + self.client_token = None + self.cookies = None + if cookie_file: + self._load_cookies(cookie_file) + + def _load_cookies(self, cookie_file: str) -> None: + """ + Load cookies from a file. + + :param cookie_file: Path to the cookie file. + """ + if not os.path.exists(cookie_file): + raise FileNotFoundError(f"Cookie file not found: {cookie_file}") + + cookie_jar = MozillaCookieJar(cookie_file) + cookie_jar.load(ignore_discard=True, ignore_expires=True) + self.cookies = requests.utils.dict_from_cookiejar(cookie_jar) + + def authenticate(self, credentials: dict) -> None: + """ + Authenticate with Spotify by fetching session data and client token. + """ + def fetch_session_data(): + url = f'https://open.spotify.com/' + headers = { + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', + } + response = requests.get(url, headers=headers) + response.raise_for_status() + soup = BeautifulSoup(response.text, 'html.parser') + session_script = soup.find('script', {'id': 'session'}) + config_script = soup.find('script', {'id': 'config'}) + if session_script and config_script: + return json.loads(session_script.string), json.loads(config_script.string) + else: + raise ValueError("Failed to fetch session or config data.") + + + self.session_data, self.config_data = fetch_session_data() + + def fetch_client_token(): + url = f'https://clienttoken.spotify.com/v1/clienttoken' + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + 'origin': 'https://open.spotify.com', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', + } + payload = { + "client_data": { + "client_version": "1.2.52.404.gcb99a997", + "client_id": self.session_data.get("clientId", ""), + "js_sdk_data": { + "device_brand": "unknown", + "device_model": "unknown", + "os": "windows", + "os_version": "NT 10.0", + "device_id": self.config_data.get("correlationId", ""), + "device_type": "computer" + } + } + } + response = requests.post(url, headers=headers, json=payload) + response.raise_for_status() + return response.json().get("granted_token", "") + + self.client_token = fetch_client_token() + + def _make_request(self, endpoint: str, params: dict = None) -> dict: + """ + Helper method to make authenticated requests to Spotify APIs. + """ + headers = { + 'accept': 'application/json', + 'app-platform': 'WebPlayer', + 'authorization': f'Bearer {self.session_data.get("accessToken", "")}', + 'client-token': self.client_token.get('token',''), + } + response = requests.get(f"{self.base_url}/{endpoint}", headers=headers, params=params) + response.raise_for_status() + return response.json() + + def get_playlist(self, playlist_id: str) -> Playlist: + """ + Fetch a playlist by ID with all tracks. + """ + limit = 50 + offset = 0 + all_items = [] + + while True: + query_parameters = { + "operationName": "fetchPlaylist", + "variables": json.dumps({ + "uri": f"spotify:playlist:{playlist_id}", + "offset": offset, + "limit": limit + }), + "extensions": json.dumps({ + "persistedQuery": { + "version": 1, + "sha256Hash": "19ff1327c29e99c208c86d7a9d8f1929cfdf3d3202a0ff4253c821f1901aa94d" + } + }) + } + encoded_query = urlencode(query_parameters) + data = self._make_request(f"pathfinder/v1/query?{encoded_query}") + playlist_data = data.get('data', {}).get('playlistV2', {}) + content = playlist_data.get('content', {}) + items = content.get('items', []) + all_items.extend(items) + + if len(all_items) >= content.get('totalCount', 0): + break + + offset += limit + + tracks = [ + Track( + id=item["itemV2"]["data"]["uri"].split(":")[-1], + name=item["itemV2"]["data"]["name"], + uri=item["itemV2"]["data"]["uri"], + duration_ms=item["itemV2"]["data"]["trackDuration"]["totalMilliseconds"], + explicit=False, # Default as Spotify API doesn't provide explicit info here + album=item["itemV2"]["data"]["albumOfTrack"]["name"], + artists=[ + artist["profile"]["name"] + for artist in item["itemV2"]["data"]["albumOfTrack"]["artists"]["items"] + ] + ) + for item in all_items + ] + return Playlist( + id=playlist_id, + name=playlist_data.get("name", ""), + description=playlist_data.get("description", ""), + uri=playlist_data.get("uri", ""), + tracks=tracks, + ) + + def search_tracks(self, query: str, limit: int = 10) -> List[Track]: + """ + Searches for tracks on Spotify. + :param query: Search query. + :param limit: Maximum number of results. + :return: A list of Track objects. + """ + print(f"search_tracks: Placeholder for search with query '{query}' and limit {limit}.") + return [] + + def get_track(self, track_id: str) -> Track: + """ + Fetches details for a specific track. + :param track_id: The ID of the track. + :return: A Track object. + """ + print(f"get_track: Placeholder for track with ID {track_id}.") + return Track(id=track_id, name="", uri="", duration_ms=0, explicit=False, album=Album(), artists=[], external_urls= ExternalUrl()) + + def get_featured_playlists(self, limit: int = 10) -> List[Playlist]: + """ + Fetches featured playlists. + :param limit: Maximum number of results. + :return: A list of Playlist objects. + """ + print(f"get_featured_playlists: Placeholder for featured playlists with limit {limit}.") + return [] + + def get_playlists_by_category(self, category_id: str, limit: int = 10) -> List[Playlist]: + """ + Fetches playlists for a specific category. + :param category_id: The ID of the category. + :param limit: Maximum number of results. + :return: A list of Playlist objects. + """ + print(f"get_playlists_by_category: Placeholder for playlists in category {category_id}.") + return [] + + def get_categories(self, limit: int = 10) -> List[Category]: + """ + Fetches categories from Spotify. + :param limit: Maximum number of results. + :return: A list of Category objects. + """ + print(f"get_categories: Placeholder for categories with limit {limit}.") + return [] \ No newline at end of file From 7232b3223d600b21dc55a000438973514977f46c Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 20:49:36 +0000 Subject: [PATCH 13/42] further implementations --- app/providers/base.py | 22 +++ app/providers/spotify.py | 353 ++++++++++++++++++++++++++++++++------- 2 files changed, 310 insertions(+), 65 deletions(-) diff --git a/app/providers/base.py b/app/providers/base.py index a49d322..9fa0956 100644 --- a/app/providers/base.py +++ b/app/providers/base.py @@ -2,6 +2,8 @@ from dataclasses import dataclass, field from typing import List, Optional from abc import ABC, abstractmethod + + @dataclass class ExternalUrl: url: str @@ -12,6 +14,26 @@ class ItemBase: name: str uri: str external_urls: Optional[List[ExternalUrl]] + +@dataclass +class Profile: + avatar: Optional[str] # Avatar URL or None + avatar_background_color: Optional[int] + name: str + uri: str + username: str + +@dataclass +class AccountAttributes: + catalogue: str + dsa_mode_available: bool + dsa_mode_enabled: bool + multi_user_plan_current_size: Optional[int] + multi_user_plan_member_type: Optional[str] + on_demand: bool + opt_in_trial_premium_only_market: bool + country: str + product: str @dataclass class Image: diff --git a/app/providers/spotify.py b/app/providers/spotify.py index 6fcf145..03ad022 100644 --- a/app/providers/spotify.py +++ b/app/providers/spotify.py @@ -1,5 +1,5 @@ import os -from app.providers.base import Album, MusicProviderClient,Playlist,Track,ExternalUrl,Category +from app.providers.base import AccountAttributes, Album, Artist, Image, MusicProviderClient, Owner,Playlist, PlaylistTrack, Profile,Track,ExternalUrl,Category import requests import json @@ -8,6 +8,9 @@ from bs4 import BeautifulSoup from urllib.parse import urlencode from typing import List, Dict, Optional from http.cookiejar import MozillaCookieJar +import logging + +l = logging.getLogger(__name__) class SpotifyClient(MusicProviderClient): """ @@ -30,62 +33,83 @@ class SpotifyClient(MusicProviderClient): :param cookie_file: Path to the cookie file. """ if not os.path.exists(cookie_file): + l.error(f"Cookie file not found: {cookie_file}") raise FileNotFoundError(f"Cookie file not found: {cookie_file}") cookie_jar = MozillaCookieJar(cookie_file) cookie_jar.load(ignore_discard=True, ignore_expires=True) self.cookies = requests.utils.dict_from_cookiejar(cookie_jar) - def authenticate(self, credentials: dict) -> None: + def authenticate(self, credentials: Optional[dict] = None) -> None: """ - Authenticate with Spotify by fetching session data and client token. + Authenticate with Spotify using cookies if available, or fetch session and config data. + + :param credentials: Optional dictionary of credentials. """ - def fetch_session_data(): - url = f'https://open.spotify.com/' - headers = { - 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', - } - response = requests.get(url, headers=headers) - response.raise_for_status() - soup = BeautifulSoup(response.text, 'html.parser') - session_script = soup.find('script', {'id': 'session'}) - config_script = soup.find('script', {'id': 'config'}) - if session_script and config_script: - return json.loads(session_script.string), json.loads(config_script.string) - else: - raise ValueError("Failed to fetch session or config data.") + if self.cookies: + l.debug("Authenticating using cookies.") + self.session_data, self.config_data = self._fetch_session_data() + self.client_token = self._fetch_client_token() + else: + l.debug("Authenticating without cookies.") + self.session_data, self.config_data = self._fetch_session_data(fetch_with_cookies=False) + self.client_token = self._fetch_client_token() - - self.session_data, self.config_data = fetch_session_data() + def _fetch_session_data(self, fetch_with_cookies: bool = True): + """ + Fetch session data from Spotify. - def fetch_client_token(): - url = f'https://clienttoken.spotify.com/v1/clienttoken' - headers = { - 'accept': 'application/json', - 'content-type': 'application/json', - 'origin': 'https://open.spotify.com', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', - } - payload = { - "client_data": { - "client_version": "1.2.52.404.gcb99a997", - "client_id": self.session_data.get("clientId", ""), - "js_sdk_data": { - "device_brand": "unknown", - "device_model": "unknown", - "os": "windows", - "os_version": "NT 10.0", - "device_id": self.config_data.get("correlationId", ""), - "device_type": "computer" - } + :param fetch_with_cookies: Whether to include cookies in the request. + :return: Tuple containing session and config data. + """ + url = 'https://open.spotify.com/' + headers = { + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', + } + cookies = self.cookies if fetch_with_cookies else None + response = requests.get(url, headers=headers, cookies=cookies) + response.raise_for_status() + soup = BeautifulSoup(response.text, 'html.parser') + session_script = soup.find('script', {'id': 'session'}) + config_script = soup.find('script', {'id': 'config'}) + if session_script and config_script: + l.debug("fetched session and config scripts") + return json.loads(session_script.string), json.loads(config_script.string) + else: + raise ValueError("Failed to fetch session or config data.") + + def _fetch_client_token(self): + """ + Fetch the client token using session data and cookies. + + :return: The client token as a string. + """ + url = f'https://clienttoken.spotify.com/v1/clienttoken' + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + 'origin': 'https://open.spotify.com', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', + } + payload = { + "client_data": { + "client_version": "1.2.52.404.gcb99a997", + "client_id": self.session_data.get("clientId", ""), + "js_sdk_data": { + "device_brand": "unknown", + "device_model": "unknown", + "os": "windows", + "os_version": "NT 10.0", + "device_id": self.config_data.get("correlationId", ""), + "device_type": "computer" } } - response = requests.post(url, headers=headers, json=payload) - response.raise_for_status() - return response.json().get("granted_token", "") - - self.client_token = fetch_client_token() + } + response = requests.post(url, headers=headers, json=payload, cookies=self.cookies) + response.raise_for_status() + l.debug("fetched granted_token") + return response.json().get("granted_token", "") def _make_request(self, endpoint: str, params: dict = None) -> dict: """ @@ -97,13 +121,115 @@ class SpotifyClient(MusicProviderClient): 'authorization': f'Bearer {self.session_data.get("accessToken", "")}', 'client-token': self.client_token.get('token',''), } - response = requests.get(f"{self.base_url}/{endpoint}", headers=headers, params=params) + l.debug(f"starting request: {self.base_url}/{endpoint}") + response = requests.get(f"{self.base_url}/{endpoint}", headers=headers, params=params, cookies=self.cookies) + response.raise_for_status() return response.json() + + #region utility functions to help parsing objects + def _parse_external_urls(self, uri: str, entity_type: str) -> List[ExternalUrl]: + """ + Create ExternalUrl instances for an entity. + + :param uri: The URI of the entity. + :param entity_type: The type of entity ('track', 'album', 'artist', 'playlist', etc.). + :return: A list of ExternalUrl instances. + """ + return [ExternalUrl(url=f"https://open.spotify.com/{entity_type}/{uri.split(':')[-1]}")] + + + def _parse_images(self, image_data: List[Dict]) -> List[Image]: + """ + Parse images from the API response. + + :param image_data: List of dictionaries containing image data. + :return: A list of Image objects. + """ + images = [] + for img in image_data: + # Extract the first source if available + sources = img.get("sources", []) + if sources: + source = sources[0] # Take the first source as the default + images.append(Image( + url=source.get("url"), + height=source.get("height"), + width=source.get("width") + )) + return images + + + def _parse_artist(self, artist_data: Dict) -> Artist: + """ + Parse an artist object from API data. + + :param artist_data: Dictionary representing an artist. + :return: An Artist instance. + """ + return Artist( + id=artist_data["uri"].split(":")[-1], + name=artist_data["profile"]["name"], + uri=artist_data["uri"], + external_urls=self._parse_external_urls(artist_data["uri"], "artist") + ) + + + def _parse_album(self, album_data: Dict) -> Album: + """ + Parse an album object from API data. + + :param album_data: Dictionary representing an album. + :return: An Album instance. + """ + return Album( + id=album_data["uri"].split(":")[-1], + name=album_data["name"], + uri=album_data["uri"], + external_urls=self._parse_external_urls(album_data["uri"], "album"), + artists=[self._parse_artist(artist) for artist in album_data["artists"]["items"]], + images=self._parse_images(album_data["coverArt"]["sources"]) + ) + + + def _parse_track(self, track_data: Dict) -> Track: + """ + Parse a track object from API data. + + :param track_data: Dictionary representing a track. + :return: A Track instance. + """ + return Track( + id=track_data["uri"].split(":")[-1], + name=track_data["name"], + uri=track_data["uri"], + external_urls=self._parse_external_urls(track_data["uri"], "track"), + duration_ms=track_data["trackDuration"]["totalMilliseconds"], + explicit=track_data.get("explicit", False), + album=self._parse_album(track_data["albumOfTrack"]), + artists=[self._parse_artist(artist) for artist in track_data["artists"]["items"]] + ) + def _parse_owner(self, owner_data: Dict) -> Optional[Owner]: + """ + Parse an owner object from API data. + + :param owner_data: Dictionary representing an owner. + :return: An Owner instance or None if the owner data is empty. + """ + if not owner_data: + return None + + return Owner( + id=owner_data.get("uri", "").split(":")[-1], + name=owner_data.get("name", ""), + uri=owner_data.get("uri", ""), + external_urls=self._parse_external_urls(owner_data.get("uri", ""), "user") + ) + #endregion def get_playlist(self, playlist_id: str) -> Playlist: """ - Fetch a playlist by ID with all tracks. + Fetch a playlist by ID with all tracks, using the defined generic classes. """ limit = 50 offset = 0 @@ -136,27 +262,34 @@ class SpotifyClient(MusicProviderClient): offset += limit - tracks = [ - Track( - id=item["itemV2"]["data"]["uri"].split(":")[-1], - name=item["itemV2"]["data"]["name"], - uri=item["itemV2"]["data"]["uri"], - duration_ms=item["itemV2"]["data"]["trackDuration"]["totalMilliseconds"], - explicit=False, # Default as Spotify API doesn't provide explicit info here - album=item["itemV2"]["data"]["albumOfTrack"]["name"], - artists=[ - artist["profile"]["name"] - for artist in item["itemV2"]["data"]["albumOfTrack"]["artists"]["items"] - ] - ) - for item in all_items - ] + # Use utility methods to parse tracks + tracks = [self._parse_track(item["itemV2"]["data"]) for item in all_items] + + images = self._parse_images(playlist_data.get("images", {}).get("items", [])) + + owner_data = playlist_data.get("ownerV2", {}).get("data", {}) + owner = self._parse_owner(owner_data) + return Playlist( id=playlist_id, name=playlist_data.get("name", ""), - description=playlist_data.get("description", ""), uri=playlist_data.get("uri", ""), - tracks=tracks, + external_urls=self._parse_external_urls(playlist_id, "playlist"), + description=playlist_data.get("description", ""), + public=playlist_data.get("public", None), + collaborative=playlist_data.get("collaborative", None), + followers=playlist_data.get("followers", 0), + images=images, + owner=owner, + tracks=[ + PlaylistTrack( + added_at=item.get("addedAt", {}).get("isoString", ""), + added_by=None, + is_local=False, + track=track + ) + for item, track in zip(all_items, tracks) + ] ) def search_tracks(self, query: str, limit: int = 10) -> List[Track]: @@ -204,4 +337,94 @@ class SpotifyClient(MusicProviderClient): :return: A list of Category objects. """ print(f"get_categories: Placeholder for categories with limit {limit}.") - return [] \ No newline at end of file + return [] + + + # non generic method implementations: + def get_profile(self) -> Optional[Profile]: + """ + Fetch the profile attributes of the authenticated Spotify user. + + :return: A Profile object containing the user's profile information or None if an error occurs. + """ + query_parameters = { + "operationName": "profileAttributes", + "variables": json.dumps({}), + "extensions": json.dumps({ + "persistedQuery": { + "version": 1, + "sha256Hash": "53bcb064f6cd18c23f752bc324a791194d20df612d8e1239c735144ab0399ced" + } + }) + } + + encoded_query = urlencode(query_parameters) + + url = f"pathfinder/v1/query?{encoded_query}" + + try: + response = self._make_request(url) + profile_data = response.get('data', {}).get('me', {}).get('profile', {}) + if not profile_data: + raise ValueError("Invalid profile data received.") + return Profile( + avatar=profile_data.get("avatar"), + avatar_background_color=profile_data.get("avatarBackgroundColor"), + name=profile_data.get("name", ""), + uri=profile_data.get("uri", ""), + username=profile_data.get("username", "") + ) + + except Exception as e: + print(f"An error occurred while fetching profile attributes: {e}") + return None + def get_account_attributes(self) -> Optional[AccountAttributes]: + """ + Fetch the account attributes of the authenticated Spotify user. + + :return: An AccountAttributes object containing the user's account information or None if an error occurs. + """ + # Define the query parameters + query_parameters = { + "operationName": "accountAttributes", + "variables": json.dumps({}), # Empty variables for this query + "extensions": json.dumps({ + "persistedQuery": { + "version": 1, + "sha256Hash": "4fbd57be3c6ec2157adcc5b8573ec571f61412de23bbb798d8f6a156b7d34cdf" + } + }) + } + + # Encode the query parameters + encoded_query = urlencode(query_parameters) + + # API endpoint + url = f"pathfinder/v1/query?{encoded_query}" + + try: + # Perform the request + response = self._make_request(url) + + # Extract and validate the account data + account_data = response.get('data', {}).get('me', {}).get('account', {}) + attributes = account_data.get("attributes", {}) + if not attributes or not account_data.get("country") or not account_data.get("product"): + raise ValueError("Invalid account data received.") + + # Map the response to the AccountAttributes class + return AccountAttributes( + catalogue=attributes.get("catalogue", ""), + dsa_mode_available=attributes.get("dsaModeAvailable", False), + dsa_mode_enabled=attributes.get("dsaModeEnabled", False), + multi_user_plan_current_size=attributes.get("multiUserPlanCurrentSize"), + multi_user_plan_member_type=attributes.get("multiUserPlanMemberType"), + on_demand=attributes.get("onDemand", False), + opt_in_trial_premium_only_market=attributes.get("optInTrialPremiumOnlyMarket", False), + country=account_data.get("country", ""), + product=account_data.get("product", "") + ) + + except Exception as e: + print(f"An error occurred while fetching account attributes: {e}") + return None \ No newline at end of file From 25e51f1ef26a96f7d666536a41962c44f114c060 Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 22:07:11 +0000 Subject: [PATCH 14/42] implemented browse_all and browse_page , should be enough for jellyplist --- app/providers/spotify.py | 217 +++++++++++++++++++++++++++++++++------ 1 file changed, 186 insertions(+), 31 deletions(-) diff --git a/app/providers/spotify.py b/app/providers/spotify.py index 03ad022..cefa375 100644 --- a/app/providers/spotify.py +++ b/app/providers/spotify.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass import os from app.providers.base import AccountAttributes, Album, Artist, Image, MusicProviderClient, Owner,Playlist, PlaylistTrack, Profile,Track,ExternalUrl,Category import requests @@ -12,6 +13,21 @@ import logging l = logging.getLogger(__name__) +@dataclass +class BrowseCard: + title: str + uri: str + background_color: str + artwork: List[Image] + + +@dataclass +class BrowseSection: + title: str + items: List[BrowseCard] + uri: str + + class SpotifyClient(MusicProviderClient): """ Spotify implementation of the MusicProviderClient. @@ -225,11 +241,97 @@ class SpotifyClient(MusicProviderClient): uri=owner_data.get("uri", ""), external_urls=self._parse_external_urls(owner_data.get("uri", ""), "user") ) + def _parse_card_artwork(self, sources: List[Dict]) -> List[Image]: + """ + Parse artwork for a browse card. + + :param sources: List of artwork source dictionaries. + :return: A list of CardArtwork instances. + """ + return [Image(url=source["url"], height=source.get("height"), width=source.get("width")) for source in sources] + + + def _parse_browse_card(self, card_data: Dict) -> BrowseCard: + """ + Parse a single browse card. + + :param card_data: Dictionary containing card data. + :return: A BrowseCard instance. + """ + card_content = card_data["content"]["data"]["data"]["cardRepresentation"] + artwork_sources = card_content["artwork"]["sources"] + + return BrowseCard( + title=card_content["title"]["transformedLabel"], + uri=card_data["uri"], + background_color=card_content["backgroundColor"]["hex"], + artwork=self._parse_card_artwork(artwork_sources) + ) + + def _parse_playlist(self, playlist_data: Dict) -> Playlist: + """ + Parse a playlist object from API response data. + + :param playlist_data: Dictionary containing playlist data. + :return: A Playlist object. + """ + images = self._parse_images(playlist_data.get("images", {}).get("items", [])) + + owner_data = playlist_data.get("ownerV2", {}).get("data", {}) + owner = self._parse_owner(owner_data) + + tracks = [ + self._parse_track(item["itemV2"]["data"]) + for item in playlist_data.get("content", {}).get("items", []) + ] + + return Playlist( + id=playlist_data.get("uri", "").split(":")[-1], + name=playlist_data.get("name", ""), + uri=playlist_data.get("uri", ""), + external_urls=self._parse_external_urls(playlist_data.get("uri", "").split(":")[-1], "playlist"), + description=playlist_data.get("description", ""), + public=playlist_data.get("public", None), + collaborative=playlist_data.get("collaborative", None), + followers=playlist_data.get("followers", 0), + images=images, + owner=owner, + tracks=[ + PlaylistTrack( + added_at=item.get("addedAt", {}).get("isoString", ""), + added_by=None, + is_local=False, + track=track + ) + for item, track in zip( + playlist_data.get("content", {}).get("items", []), + tracks + ) + ] + ) + def _parse_browse_section(self, section_data: Dict) -> BrowseSection: + """ + Parse a single browse section. + + :param section_data: Dictionary containing section data. + :return: A BrowseSection instance. + """ + section_title = section_data["data"]["title"]["transformedLabel"] + section_items = [ + item for item in section_data["sectionItems"]["items"] + if not item["uri"].startswith("spotify:xlink") + ] + return BrowseSection( + title=section_title, + items=[self._parse_browse_card(item) for item in section_items], + uri=section_data["uri"] + ) + #endregion def get_playlist(self, playlist_id: str) -> Playlist: """ - Fetch a playlist by ID with all tracks, using the defined generic classes. + Fetch a playlist by ID with all tracks. """ limit = 50 offset = 0 @@ -262,35 +364,8 @@ class SpotifyClient(MusicProviderClient): offset += limit - # Use utility methods to parse tracks - tracks = [self._parse_track(item["itemV2"]["data"]) for item in all_items] - - images = self._parse_images(playlist_data.get("images", {}).get("items", [])) - - owner_data = playlist_data.get("ownerV2", {}).get("data", {}) - owner = self._parse_owner(owner_data) - - return Playlist( - id=playlist_id, - name=playlist_data.get("name", ""), - uri=playlist_data.get("uri", ""), - external_urls=self._parse_external_urls(playlist_id, "playlist"), - description=playlist_data.get("description", ""), - public=playlist_data.get("public", None), - collaborative=playlist_data.get("collaborative", None), - followers=playlist_data.get("followers", 0), - images=images, - owner=owner, - tracks=[ - PlaylistTrack( - added_at=item.get("addedAt", {}).get("isoString", ""), - added_by=None, - is_local=False, - track=track - ) - for item, track in zip(all_items, tracks) - ] - ) + playlist_data["content"]["items"] = all_items + return self._parse_playlist(playlist_data) def search_tracks(self, query: str, limit: int = 10) -> List[Track]: """ @@ -427,4 +502,84 @@ class SpotifyClient(MusicProviderClient): except Exception as e: print(f"An error occurred while fetching account attributes: {e}") - return None \ No newline at end of file + return None + def browse_all(self, page_limit: int = 50, section_limit: int = 99) -> List[BrowseSection]: + """ + Fetch all browse sections with cards. + + :param page_limit: Maximum number of pages to fetch. + :param section_limit: Maximum number of sections per page. + :return: A list of BrowseSection objects. + """ + query_parameters = { + "operationName": "browseAll", + "variables": json.dumps({ + "pagePagination": {"offset": 0, "limit": page_limit}, + "sectionPagination": {"offset": 0, "limit": section_limit} + }), + "extensions": json.dumps({ + "persistedQuery": { + "version": 1, + "sha256Hash": "cd6fcd0ce9d1849477645646601a6d444597013355467e24066dad2c1dc9b740" + } + }) + } + encoded_query = urlencode(query_parameters) + url = f"pathfinder/v1/query?{encoded_query}" + + try: + response = self._make_request(url) + browse_data = response.get("data", {}).get("browseStart", {}).get("sections", {}) + sections = browse_data.get("items", []) + + return [self._parse_browse_section(section) for section in sections] + + except Exception as e: + print(f"An error occurred while fetching browse sections: {e}") + return [] + + def browse_page(self, card: BrowseCard) -> List[Playlist]: + """ + Fetch the content of a browse page using the URI from a BrowseCard. + + :param card: A BrowseCard instance with a URI starting with 'spotify:page'. + :return: A list of Playlist objects from the browse page. + """ + if not card.uri.startswith("spotify:page"): + raise ValueError("The BrowseCard URI must start with 'spotify:page'.") + + query_parameters = { + "operationName": "browsePage", + "variables": json.dumps({ + "pagePagination": {"offset": 0, "limit": 10}, + "sectionPagination": {"offset": 0, "limit": 10}, + "uri": card.uri + }), + "extensions": json.dumps({ + "persistedQuery": { + "version": 1, + "sha256Hash": "d8346883162a16a62a5b69e73e70c66a68c27b14265091cd9e1517f48334bbb3" + } + }) + } + encoded_query = urlencode(query_parameters) + url = f"pathfinder/v1/query?{encoded_query}" + + try: + response = self._make_request(url) + browse_data = response.get("data", {}).get("browse", {}) + sections = browse_data.get("sections", {}).get("items", []) + + playlists = [] + for section in sections: + section_items = section.get("sectionItems", {}).get("items", []) + for item in section_items: + content = item.get("content", {}).get("data", {}) + if content.get("__typename") == "Playlist": + playlists.append(self._parse_playlist(content)) + + return playlists + + except Exception as e: + print(f"An error occurred while fetching the browse page: {e}") + return [] \ No newline at end of file From 3c25cd70ea8ee20349d072de3fa66f4de95a58d5 Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 22:47:46 +0000 Subject: [PATCH 15/42] Added MusicProviderRegistry --- app/registry/__init__.py | 3 +++ app/registry/music_provider_registry.py | 35 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 app/registry/__init__.py create mode 100644 app/registry/music_provider_registry.py diff --git a/app/registry/__init__.py b/app/registry/__init__.py new file mode 100644 index 0000000..2dc5919 --- /dev/null +++ b/app/registry/__init__.py @@ -0,0 +1,3 @@ +from .music_provider_registry import MusicProviderRegistry + +__all__ = ["MusicProviderRegistry"] \ No newline at end of file diff --git a/app/registry/music_provider_registry.py b/app/registry/music_provider_registry.py new file mode 100644 index 0000000..89aeaf4 --- /dev/null +++ b/app/registry/music_provider_registry.py @@ -0,0 +1,35 @@ +from app.providers.base import MusicProviderClient + + +class MusicProviderRegistry: + """ + Registry to manage and retrieve music provider clients. + """ + _providers = {} + + @classmethod + def register_provider(cls, provider: MusicProviderClient): + """ + Registers a music provider client instance. + :param provider: An instance of a MusicProviderClient subclass. + """ + cls._providers[provider._identifier] = provider + + @classmethod + def get_provider(cls, identifier: str) -> MusicProviderClient: + """ + Retrieves a registered music provider client by its identifier. + :param identifier: The unique identifier for the provider. + :return: An instance of MusicProviderClient. + """ + if identifier not in cls._providers: + raise ValueError(f"No provider found with identifier '{identifier}'.") + return cls._providers[identifier] + + @classmethod + def list_providers(cls) -> list: + """ + Lists all registered providers. + :return: A list of registered provider identifiers. + """ + return list(cls._providers.keys()) From 33ccbc470cc1f2f2eb2ea2a8c4fef33b2cb81a26 Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 22:48:06 +0000 Subject: [PATCH 16/42] Added Identifier to base and implementation --- app/providers/base.py | 10 +++++++++- app/providers/spotify.py | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/providers/base.py b/app/providers/base.py index 9fa0956..6968a80 100644 --- a/app/providers/base.py +++ b/app/providers/base.py @@ -88,7 +88,15 @@ class MusicProviderClient(ABC): """ Abstract base class defining the interface for music provider clients. """ - + @property + @abstractmethod + def _identifier(self) -> str: + """ + A unique identifier for the music provider. + Must be implemented by all subclasses. + """ + pass + @abstractmethod def authenticate(self, credentials: dict) -> None: """ diff --git a/app/providers/spotify.py b/app/providers/spotify.py index cefa375..83f1f06 100644 --- a/app/providers/spotify.py +++ b/app/providers/spotify.py @@ -32,7 +32,10 @@ class SpotifyClient(MusicProviderClient): """ Spotify implementation of the MusicProviderClient. """ - + @property + def _identifier(self) -> str: + return "Spotify" + def __init__(self, cookie_file: Optional[str] = None): self.base_url = "https://api-partner.spotify.com" self.session_data = None From aa718eb62872812646b96b9865f27ec064fe6501 Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 22:48:29 +0000 Subject: [PATCH 17/42] Typings pyi --- app/providers/__init__.py | 3 +++ app/typings.pyi | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 app/providers/__init__.py create mode 100644 app/typings.pyi diff --git a/app/providers/__init__.py b/app/providers/__init__.py new file mode 100644 index 0000000..c8f2eb9 --- /dev/null +++ b/app/providers/__init__.py @@ -0,0 +1,3 @@ +from .spotify import SpotifyClient + +__all__ = ["SpotifyClient"] \ No newline at end of file diff --git a/app/typings.pyi b/app/typings.pyi new file mode 100644 index 0000000..9549914 --- /dev/null +++ b/app/typings.pyi @@ -0,0 +1,7 @@ +from flask import g +from providers.base import MusicProviderClient + +g: "Global" + +class Global: + music_provider: MusicProviderClient \ No newline at end of file From 56aaec603bbf1728bab15ff61d5b5a68df06dfab Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 22:48:46 +0000 Subject: [PATCH 18/42] refactor to start working with blueprints --- app/__init__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 33a62bc..841606d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -21,6 +21,7 @@ from flask_caching import Cache from .version import __version__ + def check_db_connection(db_uri, retries=5, delay=5): """ Check if the database is reachable. @@ -161,12 +162,21 @@ celery.set_default() app.logger.info(f'Jellyplist {__version__}{read_dev_build_file()} started') app.logger.debug(f"Debug logging active") from app import routes +app.register_blueprint(routes.pl_bp) + from app import jellyfin_routes, tasks if "worker" in sys.argv: tasks.release_lock("download_missing_tracks_lock") - + from app import filters # Import the filters dictionary # Register all filters for name, func in filters.filters.items(): - app.jinja_env.filters[name] = func \ No newline at end of file + app.jinja_env.filters[name] = func + + +from .providers import SpotifyClient +spotify_client = SpotifyClient('/jellyplist/open.spotify.com_cookies.txt') +spotify_client.authenticate() +from .registry import MusicProviderRegistry +MusicProviderRegistry.register_provider(spotify_client) From 94d401a99ffd45183e8f98dbffeb824014c47f67 Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 22:50:10 +0000 Subject: [PATCH 19/42] changed "spotify" to "provider" --- app/functions.py | 146 ++++++++++++++++++++++++++--------------- app/jellyfin_routes.py | 50 ++++++++------ app/models.py | 12 ++-- app/routes.py | 84 ++++++++++++++++++++---- app/tasks.py | 32 ++++----- 5 files changed, 217 insertions(+), 107 deletions(-) diff --git a/app/functions.py b/app/functions.py index 7eed5ac..3144ac0 100644 --- a/app/functions.py +++ b/app/functions.py @@ -1,4 +1,7 @@ +import json +from typing import Optional from flask import flash, redirect, session, url_for +import requests from app.models import JellyfinUser, Playlist,Track from app import sp, cache, app, jellyfin ,jellyfin_admin_token, jellyfin_admin_id,device_id, cache from functools import wraps @@ -57,51 +60,57 @@ def prepPlaylistData(data): for playlist_data in data['playlists']['items']: # Fetch the playlist from the database if it exists - db_playlist = Playlist.query.filter_by(spotify_playlist_id=playlist_data['id']).first() + if playlist_data: + db_playlist = Playlist.query.filter_by(provider_playlist_id=playlist_data['id']).first() - if db_playlist: - # If the playlist is in the database, use the stored values - if isinstance(playlist_data['tracks'],list): - track_count = len(playlist_data['tracks'] ) + if db_playlist: + # If the playlist is in the database, use the stored values + if playlist_data.get('tracks'): + if isinstance(playlist_data['tracks'],list): + track_count = len(playlist_data['tracks'] ) + else: + track_count = playlist_data['tracks']['total'] or 0 + else: + track_count = 0 + tracks_available = db_playlist.tracks_available or 0 + tracks_linked = len([track for track in db_playlist.tracks if track.jellyfin_id]) or 0 + percent_available = (tracks_available / track_count * 100) if track_count > 0 else 0 + + # Determine playlist status + if not playlist_data.get('status'): + if tracks_available == track_count and track_count > 0: + playlist_data['status'] = 'green' # Fully available + elif tracks_available > 0: + playlist_data['status'] = 'yellow' # Partially available + else: + playlist_data['status'] = 'red' # Not available + else: - track_count = playlist_data['tracks']['total'] or 0 - tracks_available = db_playlist.tracks_available or 0 - tracks_linked = len([track for track in db_playlist.tracks if track.jellyfin_id]) or 0 - percent_available = (tracks_available / track_count * 100) if track_count > 0 else 0 - - # Determine playlist status - if tracks_available == track_count and track_count > 0: - status = 'green' # Fully available - elif tracks_available > 0: - status = 'yellow' # Partially available - else: - status = 'red' # Not available - else: - # If the playlist is not in the database, initialize with 0 - track_count = 0 - tracks_available = 0 - tracks_linked = 0 - percent_available = 0 - status = 'red' # Not requested yet + # If the playlist is not in the database, initialize with 0 + track_count = 0 + tracks_available = 0 + tracks_linked = 0 + percent_available = 0 + playlist_data['status'] = 'red' # Not requested yet - # Append playlist data to the list - playlists.append({ - 'name': playlist_data['name'], - 'description': playlist_data['description'], - 'image': playlist_data['images'][0]['url'] if playlist_data['images'] else 'default-image.jpg', - 'url': playlist_data['external_urls']['spotify'], - 'id': playlist_data['id'], - 'jellyfin_id': db_playlist.jellyfin_id if db_playlist else '', - 'can_add': (db_playlist not in jellyfin_user.playlists) if db_playlist else True, - 'can_remove' : (db_playlist in jellyfin_user.playlists) if db_playlist else False, - 'last_updated':db_playlist.last_updated if db_playlist else '', - 'last_changed':db_playlist.last_changed if db_playlist else '', - 'tracks_available': tracks_available, - 'track_count': track_count, - 'tracks_linked': tracks_linked, - 'percent_available': percent_available, - 'status': status # Red, yellow, or green based on availability - }) + # Append playlist data to the list + playlists.append({ + 'name': playlist_data['name'], + 'description': playlist_data['description'], + 'image': playlist_data['images'][0]['url'] if playlist_data.get('images') else '/static/images/placeholder.png', + 'url': playlist_data['external_urls']['spotify'] if playlist_data.get('external_urls') else '', + 'id': playlist_data['id'] if playlist_data['id'] else '', + 'jellyfin_id': db_playlist.jellyfin_id if db_playlist else '', + 'can_add': (db_playlist not in jellyfin_user.playlists) if db_playlist else True, + 'can_remove' : (db_playlist in jellyfin_user.playlists) if db_playlist else False, + 'last_updated':db_playlist.last_updated if db_playlist else '', + 'last_changed':db_playlist.last_changed if db_playlist else '', + 'tracks_available': tracks_available, + 'track_count': track_count, + 'tracks_linked': tracks_linked, + 'percent_available': percent_available, + 'status': playlist_data['status'] # Red, yellow, or green based on availability + }) return playlists @@ -115,24 +124,57 @@ def get_cached_spotify_playlists(playlist_ids): spotify_data = {'playlists': {'items': []}} for playlist_id in playlist_ids: - playlist_data = get_cached_spotify_playlist(playlist_id) + playlist_data = None + not_found = False + try: + playlist_data = get_cached_spotify_playlist(playlist_id) + + except SpotifyException as e: + app.logger.error(f"Error Fetching Playlist {playlist_id}: {e}") + not_found = 'http status: 404' in str(e) + if not_found: + playlist_data = { + 'status':'red', + 'description': 'Playlist has most likely been removed. You can keep it, but won´t receive Updates.', + 'id': playlist_id, + 'name' : '' + + } + if playlist_data: spotify_data['playlists']['items'].append(playlist_data) - else: - app.logger.warning(f"Playlist data for ID {playlist_id} could not be retrieved.") return spotify_data -@cache.memoize(timeout=3600) -def get_cached_spotify_playlist(playlist_id): +@cache.memoize(timeout=3600) +def get_cached_playlist(playlist_id): """ Fetches a Spotify playlist by its ID, utilizing caching to minimize API calls. :param playlist_id: The Spotify playlist ID. :return: Playlist data as a dictionary, or None if an error occurs. """ - playlist_data = sp.playlist(playlist_id) # Fetch data from Spotify API - return playlist_data + # When the playlist_id starts with 37i9dQZF1, we need to use the new function + # as the standard Spotify API endpoints are deprecated for these playlists. + # Reference: https://github.com/kamilkosek/jellyplist/issues/25 + + if playlist_id.startswith("37i9dQZF1"): + app.logger.warning(f"Algorithmic or Spotify-owned editorial playlist, using custom Implementation to fetch details") + # Use the custom implementation for these playlists + try: + data = fetch_spotify_playlist(playlist_id) + return transform_playlist_response(data) + except Exception as e: + print(f"Error fetching playlist with custom method: {e}") + return None + + # Otherwise, use the standard Spotipy API + try: + playlist_data = sp.playlist(playlist_id) # Fetch data using Spotipy + return playlist_data + except Exception as e: + print(f"Error fetching playlist with Spotipy: {e}") + return None @cache.memoize(timeout=3600*24*10) def get_cached_spotify_track(track_id): @@ -189,7 +231,7 @@ def getFeaturedPlaylists(country: str, offset: int): def getCategoryPlaylists(category: str, offset: int): try: - playlists_data = sp.category_playlists(category_id=category, limit=16, offset=offset) + playlists_data = sp.category_playlists(category_id=category, country=app.config['SPOTIFY_COUNTRY_CODE'], limit=16, offset=offset) return prepPlaylistData(playlists_data), playlists_data['playlists']['total'], f"Category {playlists_data['message']}" except SpotifyException as e: app.logger.error(f"Spotify API error in getCategoryPlaylists: {e}") @@ -215,14 +257,14 @@ def get_tracks_for_playlist(data): tracks = [] is_admin = session.get('is_admin', False) - for idx, item in enumerate(results['tracks']): + for idx, item in enumerate(results['tracks']['items']): track_data = item['track'] if track_data: duration_ms = track_data['duration_ms'] minutes = duration_ms // 60000 seconds = (duration_ms % 60000) // 1000 - track_db = Track.query.filter_by(spotify_track_id=track_data['id']).first() + track_db = Track.query.filter_by(provider_track_id=track_data['id']).first() if track_db: downloaded = track_db.downloaded diff --git a/app/jellyfin_routes.py b/app/jellyfin_routes.py index 2190c53..74c3ba5 100644 --- a/app/jellyfin_routes.py +++ b/app/jellyfin_routes.py @@ -16,25 +16,37 @@ def jellyfin_playlists(): try: # Fetch playlists from Jellyfin playlists = jellyfin.get_playlists(session_token=functions._get_token_from_sessioncookie()) - + spotify_data = {'playlists': {'items': []}} + # Extract Spotify playlist IDs from the database - spotify_playlist_ids = [] for pl in playlists: # Retrieve the playlist from the database using Jellyfin ID from_db = Playlist.query.filter_by(jellyfin_id=pl['Id']).first() - if from_db and from_db.spotify_playlist_id: - spotify_playlist_ids.append(from_db.spotify_playlist_id) + playlist_data = None + not_found = False + if from_db and from_db.provider_playlist_id: + pl_id = from_db.provider_playlist_id + try: + playlist_data = functions.get_cached_spotify_playlist(pl_id) + + except SpotifyException as e: + app.logger.error(f"Error Fetching Playlist {pl_id}: {e}") + not_found = 'http status: 404' in str(e) + if not_found: + playlist_data = { + 'status':'red', + 'description': 'Playlist has most likely been removed. You can keep it, but won´t receive Updates.', + 'id': from_db.provider_playlist_id, + 'name' : from_db.name + + } + + if playlist_data: + spotify_data['playlists']['items'].append(playlist_data) + else: app.logger.warning(f"No database entry found for Jellyfin playlist ID: {pl['Id']}") - if not spotify_playlist_ids: - flash('No Spotify playlists found to display.', 'warning') - return render_template('jellyfin_playlists.html', playlists=functions.prepPlaylistData({'playlists': {'items': []}})) - - # Use the cached function to fetch Spotify playlists - spotify_data = functions.get_cached_spotify_playlists(spotify_playlist_ids) - - # Prepare the data for the template prepared_data = functions.prepPlaylistData(spotify_data) return render_template('jellyfin_playlists.html', playlists=prepared_data) @@ -63,13 +75,13 @@ def add_playlist(): playlist_data = functions.get_cached_spotify_playlist(playlist_id) # Check if playlist already exists in the database - playlist = Playlist.query.filter_by(spotify_playlist_id=playlist_id).first() + playlist = Playlist.query.filter_by(provider_playlist_id=playlist_id).first() if not playlist: # Add new playlist if it doesn't exist # create the playlist via api key, with the first admin as 'owner' fromJellyfin = jellyfin.create_music_playlist(functions._get_api_token(),playlist_data['name'],[],functions._get_admin_id())['Id'] - playlist = Playlist(name=playlist_data['name'], spotify_playlist_id=playlist_id,spotify_uri=playlist_data['uri'],track_count = playlist_data['tracks']['total'], tracks_available=0, jellyfin_id = fromJellyfin) + playlist = Playlist(name=playlist_data['name'], provider_playlist_id=playlist_id,provider_uri=playlist_data['uri'],track_count = playlist_data['tracks']['total'], tracks_available=0, jellyfin_id = fromJellyfin) db.session.add(playlist) db.session.commit() if app.config['START_DOWNLOAD_AFTER_PLAYLIST_ADD']: @@ -83,7 +95,7 @@ def add_playlist(): spotify_tracks = {} offset = 0 while True: - playlist_items = sp.playlist_items(playlist.spotify_playlist_id, offset=offset, limit=100) + playlist_items = sp.playlist_items(playlist.provider_playlist_id, offset=offset, limit=100) items = playlist_items['items'] spotify_tracks.update({offset + idx: track['track'] for idx, track in enumerate(items) if track['track']}) @@ -94,11 +106,11 @@ def add_playlist(): track_info = track_data if not track_info: continue - track = Track.query.filter_by(spotify_track_id=track_info['id']).first() + track = Track.query.filter_by(provider_track_id=track_info['id']).first() if not track: # Add new track if it doesn't exist - track = Track(name=track_info['name'], spotify_track_id=track_info['id'], spotify_uri=track_info['uri'], downloaded=False) + track = Track(name=track_info['name'], provider_track_id=track_info['id'], provider_uri=track_info['uri'], downloaded=False) db.session.add(track) db.session.commit() elif track.downloaded: @@ -158,7 +170,7 @@ def delete_playlist(playlist_id): flash('Playlist removed') item = { "name" : playlist.name, - "id" : playlist.spotify_playlist_id, + "id" : playlist.provider_playlist_id, "can_add":True, "can_remove":False, "jellyfin_id" : playlist.jellyfin_id @@ -182,7 +194,7 @@ def wipe_playlist(playlist_id): if playlist: # Delete the playlist name = playlist.name - id = playlist.spotify_playlist_id + id = playlist.provider_playlist_id jf_id = playlist.jellyfin_id db.session.delete(playlist) db.session.commit() diff --git a/app/models.py b/app/models.py index 12d76db..76bac0f 100644 --- a/app/models.py +++ b/app/models.py @@ -22,8 +22,8 @@ user_playlists = db.Table('user_playlists', class Playlist(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(150), nullable=False) - spotify_playlist_id = db.Column(db.String(120), unique=True, nullable=False) - spotify_uri = db.Column(db.String(120), unique=True, nullable=False) + provider_playlist_id = db.Column(db.String(120), unique=True, nullable=False) + provider_uri = db.Column(db.String(120), unique=True, nullable=False) # Relationship with Tracks tracks = db.relationship('Track', secondary='playlist_tracks', back_populates='playlists') @@ -37,7 +37,7 @@ class Playlist(db.Model): users = db.relationship('JellyfinUser', secondary=user_playlists, back_populates='playlists') def __repr__(self): - return f'' + return f'' # Association table between Playlists and Tracks playlist_tracks = db.Table('playlist_tracks', @@ -50,8 +50,8 @@ playlist_tracks = db.Table('playlist_tracks', class Track(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(150), nullable=False) - spotify_track_id = db.Column(db.String(120), unique=True, nullable=False) - spotify_uri = db.Column(db.String(120), unique=True, nullable=False) + provider_track_id = db.Column(db.String(120), unique=True, nullable=False) + provider_uri = db.Column(db.String(120), unique=True, nullable=False) downloaded = db.Column(db.Boolean()) filesystem_path = db.Column(db.String(), nullable=True) jellyfin_id = db.Column(db.String(120), nullable=True) # Add Jellyfin track ID field @@ -60,4 +60,4 @@ class Track(db.Model): # Many-to-Many relationship with Playlists playlists = db.relationship('Playlist', secondary=playlist_tracks, back_populates='tracks') def __repr__(self): - return f'' + return f'' diff --git a/app/routes.py b/app/routes.py index a346f7d..fcf0cb1 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,10 +1,29 @@ -from flask import Flask, Response, jsonify, render_template, request, redirect, url_for, session, flash +import json +import re +from flask import Flask, Response, jsonify, render_template, request, redirect, url_for, session, flash, Blueprint, g from app import app, db, functions, sp, jellyfin, celery, jellyfin_admin_token, jellyfin_admin_id,device_id, cache, read_dev_build_file, tasks from app.models import JellyfinUser,Playlist,Track from celery.result import AsyncResult + +from app.providers import base +from app.providers.base import MusicProviderClient +from app.providers.spotify import SpotifyClient +from app.registry.music_provider_registry import MusicProviderRegistry from .version import __version__ from spotipy.exceptions import SpotifyException +pl_bp = Blueprint('playlist', __name__) +@pl_bp.before_request +def set_active_provider(): + """ + Middleware to select the active provider based on request parameters. + """ + provider_id = request.args.get('provider', 'Spotify') # Default to Spotify + try: + g.music_provider = MusicProviderRegistry.get_provider(provider_id) + except ValueError as e: + return {"error": str(e)}, 400 + @app.context_processor def add_context(): unlinked_track_count = len(Track.query.filter_by(downloaded=True,jellyfin_id=None).all()) @@ -45,7 +64,7 @@ def link_issues(): unlinked_tracks = Track.query.filter_by(downloaded=True,jellyfin_id=None).all() tracks = [] for ult in unlinked_tracks: - sp_track = functions.get_cached_spotify_track(ult.spotify_track_id) + sp_track = functions.get_cached_spotify_track(ult.provider_track_id) duration_ms = sp_track['duration_ms'] minutes = duration_ms // 60000 seconds = (duration_ms % 60000) // 1000 @@ -117,7 +136,7 @@ def login(): db.session.add(new_user) db.session.commit() - return redirect('/playlists') + return redirect('/') except: flash('Login failed. Please check your Jellyfin credentials and try again.', 'error') return redirect(url_for('login')) @@ -130,6 +149,32 @@ def logout(): session.pop('jellyfin_access_token', None) return redirect(url_for('login')) +@app.route('/add_single',methods=['GET']) +@functions.jellyfin_login_required +def add_single(): + playlist = request.args.get('playlist') + error = None + errdata= None + if playlist: + parsed = sp._get_id(type='playlist',id=playlist) + if parsed: + try: + functions.get_cached_spotify_playlist(parsed) + + return redirect(f'/playlist/view/{parsed}') + except SpotifyException as e: + url_match = re.search(sp._regex_spotify_url, playlist) + if url_match is not None: + resp = functions.fetch_spotify_playlist(playlist,None) + parsed_data = functions.parse_spotify_playlist_html(resp) + error = (f'Playlist can´t be fetched') + + errdata = str(e) + + return render_template('index.html',error_message = error, error_data = errdata) + + + @app.route('/playlists') @app.route('/categories') @@ -147,8 +192,13 @@ def loaditems(): try: db_playlists = db.session.query(Playlist).offset(offset).limit(limit).all() max_items = db.session.query(Playlist).count() - spotify_playlist_ids = [playlist.spotify_playlist_id for playlist in db_playlists] - spotify_data = functions.get_cached_spotify_playlists(tuple(spotify_playlist_ids)) + + provider_playlist_ids = [playlist.provider_playlist_id for playlist in db_playlists] + spotify_data = functions.get_cached_spotify_playlists(tuple(provider_playlist_ids)) + for x in spotify_data['playlists']['items']: + for from_db in db_playlists: + if x['id'] == from_db.provider_playlist_id: + x['name'] = from_db.name data = functions.prepPlaylistData(spotify_data) items_title = "Monitored Playlists" items_subtitle = "These playlists are already monitored by the Server. If you add one to your Jellyfin account, they will be available immediately." @@ -209,11 +259,11 @@ def searchResults(): context = {} if query: # Add your logic here to perform the search on Spotify (or Jellyfin) - search_result = sp.search(q = query, type= 'track,album,artist,playlist') + search_result = sp.search(q = query, type= 'playlist',limit= 50, market=app.config['SPOTIFY_COUNTRY_CODE']) context = { - 'artists' : functions.prepArtistData(search_result ), + #'artists' : functions.prepArtistData(search_result ), 'playlists' : functions.prepPlaylistData(search_result ), - 'albums' : functions.prepAlbumData(search_result ), + #'albums' : functions.prepAlbumData(search_result ), 'query' : query } return render_template('search.html', **context) @@ -221,14 +271,14 @@ def searchResults(): return render_template('search.html', query=None, results={}) -@app.route('/playlist/view/') +@pl_bp.route('/playlist/view/') @functions.jellyfin_login_required def get_playlist_tracks(playlist_id): - # Hol dir alle Tracks für die Playlist - data = functions.get_full_playlist_data(playlist_id) # Diese neue Funktion holt alle Tracks der Playlist - tracks = functions.get_tracks_for_playlist(data) # Deine Funktion, um Tracks zu holen + provider: MusicProviderClient = g.music_provider # Explicit type hint for g.music_provider + playlist: base.Playlist = provider.get_playlist(playlist_id) + tracks = functions.get_tracks_for_playlist(playlist.tracks) # Deine Funktion, um Tracks zu holen # Berechne die gesamte Dauer der Playlist - total_duration_ms = sum([track['track']['duration_ms'] for track in data['tracks'] if track['track']]) + total_duration_ms = sum([track['track']['duration_ms'] for track in data['tracks']['items'] if track['track']]) # Konvertiere die Gesamtdauer in ein lesbares Format hours, remainder = divmod(total_duration_ms // 1000, 3600) @@ -263,7 +313,7 @@ def associate_track(): flash('Missing Jellyfin or Spotify ID') # Retrieve the track by Spotify ID - track = Track.query.filter_by(spotify_track_id=spotify_id).first() + track = Track.query.filter_by(provider_track_id=spotify_id).first() if not track: flash('Track not found') @@ -296,4 +346,10 @@ def unlock_key(): @app.route('/test') def test(): + playlist_id = "37i9dQZF1DX12qgyzUprB6" + client = SpotifyClient(cookie_file='/jellyplist/open.spotify.com_cookies.txt') + client.authenticate() + pl = client.get_playlist(playlist_id=playlist_id) + browse = client.browse_all() + page = client.browse_page(browse[0].items[12]) return '' \ No newline at end of file diff --git a/app/tasks.py b/app/tasks.py index 815b399..a8084d6 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -49,7 +49,7 @@ def update_all_playlists_track_status(self): for playlist in playlists: total_tracks = 0 available_tracks = 0 - app.logger.debug(f"Current Playlist: {playlist.name} [{playlist.id}:{playlist.spotify_playlist_id}]" ) + app.logger.debug(f"Current Playlist: {playlist.name} [{playlist.id}:{playlist.provider_playlist_id}]" ) for track in playlist.tracks: total_tracks += 1 if track.filesystem_path and os.path.exists(track.filesystem_path): @@ -106,21 +106,21 @@ def download_missing_tracks(self): processed_tracks = 0 failed_downloads = 0 for track in undownloaded_tracks: - app.logger.info(f"Processing track: {track.name} [{track.spotify_track_id}]") + app.logger.info(f"Processing track: {track.name} [{track.provider_track_id}]") # Check if the track already exists in the output directory - file_path = f"{output_dir.replace('{track-id}', track.spotify_track_id)}.mp3" + file_path = f"{output_dir.replace('{track-id}', track.provider_track_id)}.mp3" # region search before download if search_before_download: app.logger.info(f"Searching for track in Jellyfin: {track.name}") - spotify_track = functions.get_cached_spotify_track(track.spotify_track_id) + spotify_track = functions.get_cached_spotify_track(track.provider_track_id) # at first try to find the track without fingerprinting it best_match = find_best_match_from_jellyfin(track) if best_match: track.downloaded = True if track.jellyfin_id != best_match['Id']: track.jellyfin_id = best_match['Id'] - app.logger.info(f"Updated Jellyfin ID for track: {track.name} ({track.spotify_track_id})") + app.logger.info(f"Updated Jellyfin ID for track: {track.name} ({track.provider_track_id})") if track.filesystem_path != best_match['Path']: track.filesystem_path = best_match['Path'] @@ -171,8 +171,8 @@ def download_missing_tracks(self): # Attempt to download the track using spotdl try: - app.logger.info(f"Trying to download track: {track.name} ({track.spotify_track_id}), spotdl timeout = 90") - s_url = f"https://open.spotify.com/track/{track.spotify_track_id}" + app.logger.info(f"Trying to download track: {track.name} ({track.provider_track_id}), spotdl timeout = 90") + s_url = f"https://open.spotify.com/track/{track.provider_track_id}" command = [ "spotdl", "download", s_url, @@ -251,7 +251,7 @@ def check_for_playlist_updates(self): for playlist in playlists: playlist.last_updated = datetime.now( timezone.utc) - sp_playlist = sp.playlist(playlist.spotify_playlist_id) + sp_playlist = sp.playlist(playlist.provider_playlist_id) full_update = True app.logger.info(f'Checking updates for playlist: {playlist.name}, s_snapshot = {sp_playlist['snapshot_id']}') db.session.commit() @@ -266,7 +266,7 @@ def check_for_playlist_updates(self): offset = 0 playlist.snapshot_id = sp_playlist['snapshot_id'] while True: - playlist_data = sp.playlist_items(playlist.spotify_playlist_id, offset=offset, limit=100) + playlist_data = sp.playlist_items(playlist.provider_playlist_id, offset=offset, limit=100) items = playlist_data['items'] spotify_tracks.update({offset + idx: track['track'] for idx, track in enumerate(items) if track['track']}) @@ -274,7 +274,7 @@ def check_for_playlist_updates(self): break offset += 100 # Move to the next batch - existing_tracks = {track.spotify_track_id: track for track in playlist.tracks} + existing_tracks = {track.provider_track_id: track for track in playlist.tracks} # Determine tracks to add and remove tracks_to_add = [] @@ -282,9 +282,9 @@ def check_for_playlist_updates(self): if track_info: track_id = track_info['id'] if track_id not in existing_tracks: - track = Track.query.filter_by(spotify_track_id=track_id).first() + track = Track.query.filter_by(provider_track_id=track_id).first() if not track: - track = Track(name=track_info['name'], spotify_track_id=track_id, spotify_uri=track_info['uri'], downloaded=False) + track = Track(name=track_info['name'], provider_track_id=track_id, provider_uri=track_info['uri'], downloaded=False) db.session.add(track) db.session.commit() app.logger.info(f'Added new track: {track.name}') @@ -382,10 +382,10 @@ def update_jellyfin_id_for_downloaded_tracks(self): track.downloaded = True if track.jellyfin_id != best_match['Id']: track.jellyfin_id = best_match['Id'] - app.logger.info(f"Updated Jellyfin ID for track: {track.name} ({track.spotify_track_id})") + app.logger.info(f"Updated Jellyfin ID for track: {track.name} ({track.provider_track_id})") if track.filesystem_path != best_match['Path']: track.filesystem_path = best_match['Path'] - app.logger.info(f"Updated filesystem_path for track: {track.name} ({track.spotify_track_id})") + app.logger.info(f"Updated filesystem_path for track: {track.name} ({track.provider_track_id})") @@ -422,10 +422,10 @@ def find_best_match_from_jellyfin(track: Track): for result in search_results: - app.logger.debug(f"Processing search result: {result['Id']}") + app.logger.debug(f"Processing search result: {result['Id']}, Path = {result['Path']}") quality_score = compute_quality_score(result, app.config['FIND_BEST_MATCH_USE_FFPROBE']) try: - spotify_track = functions.get_cached_spotify_track(track.spotify_track_id) + spotify_track = functions.get_cached_spotify_track(track.provider_track_id) spotify_track_name = spotify_track['name'].lower() spotify_artists = [artist['name'].lower() for artist in spotify_track['artists']] except Exception as e: From e4286299280078b0f397435b062672804706bb7b Mon Sep 17 00:00:00 2001 From: Kamil Date: Fri, 29 Nov 2024 22:54:22 +0000 Subject: [PATCH 20/42] refacotring db to work with multiple music provider --- ...9_refacotring_db_to_work_with_multiple_.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 migrations/versions/18d056f49f59_refacotring_db_to_work_with_multiple_.py diff --git a/migrations/versions/18d056f49f59_refacotring_db_to_work_with_multiple_.py b/migrations/versions/18d056f49f59_refacotring_db_to_work_with_multiple_.py new file mode 100644 index 0000000..25b1f36 --- /dev/null +++ b/migrations/versions/18d056f49f59_refacotring_db_to_work_with_multiple_.py @@ -0,0 +1,70 @@ +"""refacotring db to work with multiple music provider + +Revision ID: 18d056f49f59 +Revises: d4fef99d5d3c +Create Date: 2024-11-29 22:51:41.271688 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '18d056f49f59' +down_revision = 'd4fef99d5d3c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('playlist', schema=None) as batch_op: + batch_op.add_column(sa.Column('provider_playlist_id', sa.String(length=120), nullable=False)) + batch_op.add_column(sa.Column('provider_uri', sa.String(length=120), nullable=False)) + batch_op.add_column(sa.Column('provider_id', sa.String(length=20), nullable=True)) + batch_op.drop_constraint('playlist_spotify_playlist_id_key', type_='unique') + batch_op.drop_constraint('playlist_spotify_uri_key', type_='unique') + batch_op.create_unique_constraint(None, ['provider_uri']) + batch_op.create_unique_constraint(None, ['provider_playlist_id']) + batch_op.drop_column('spotify_playlist_id') + batch_op.drop_column('spotify_uri') + + with op.batch_alter_table('track', schema=None) as batch_op: + batch_op.add_column(sa.Column('provider_track_id', sa.String(length=120), nullable=False)) + batch_op.add_column(sa.Column('provider_uri', sa.String(length=120), nullable=False)) + batch_op.add_column(sa.Column('provider_id', sa.String(length=20), nullable=True)) + batch_op.drop_constraint('track_spotify_track_id_key', type_='unique') + batch_op.drop_constraint('track_spotify_uri_key', type_='unique') + batch_op.create_unique_constraint(None, ['provider_track_id']) + batch_op.create_unique_constraint(None, ['provider_uri']) + batch_op.drop_column('spotify_track_id') + batch_op.drop_column('spotify_uri') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('track', schema=None) as batch_op: + batch_op.add_column(sa.Column('spotify_uri', sa.VARCHAR(length=120), autoincrement=False, nullable=False)) + batch_op.add_column(sa.Column('spotify_track_id', sa.VARCHAR(length=120), autoincrement=False, nullable=False)) + batch_op.drop_constraint(None, type_='unique') + batch_op.drop_constraint(None, type_='unique') + batch_op.create_unique_constraint('track_spotify_uri_key', ['spotify_uri']) + batch_op.create_unique_constraint('track_spotify_track_id_key', ['spotify_track_id']) + batch_op.drop_column('provider_id') + batch_op.drop_column('provider_uri') + batch_op.drop_column('provider_track_id') + + with op.batch_alter_table('playlist', schema=None) as batch_op: + batch_op.add_column(sa.Column('spotify_uri', sa.VARCHAR(length=120), autoincrement=False, nullable=False)) + batch_op.add_column(sa.Column('spotify_playlist_id', sa.VARCHAR(length=120), autoincrement=False, nullable=False)) + batch_op.drop_constraint(None, type_='unique') + batch_op.drop_constraint(None, type_='unique') + batch_op.create_unique_constraint('playlist_spotify_uri_key', ['spotify_uri']) + batch_op.create_unique_constraint('playlist_spotify_playlist_id_key', ['spotify_playlist_id']) + batch_op.drop_column('provider_id') + batch_op.drop_column('provider_uri') + batch_op.drop_column('provider_playlist_id') + + # ### end Alembic commands ### From d70c3b39139ea31b9b51ff390e3ed465e86b62bd Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:32:11 +0000 Subject: [PATCH 21/42] Major Overhaul: Cleanup Unused Files --- app/jellyfin_routes.py | 228 ---------------- app/routes.py | 355 ------------------------- templates/partials/_spotify_item.html | 49 ---- templates/partials/_spotify_items.html | 24 -- templates/partials/_toast.html | 18 -- 5 files changed, 674 deletions(-) delete mode 100644 app/jellyfin_routes.py delete mode 100644 app/routes.py delete mode 100644 templates/partials/_spotify_item.html delete mode 100644 templates/partials/_spotify_items.html delete mode 100644 templates/partials/_toast.html diff --git a/app/jellyfin_routes.py b/app/jellyfin_routes.py deleted file mode 100644 index 74c3ba5..0000000 --- a/app/jellyfin_routes.py +++ /dev/null @@ -1,228 +0,0 @@ -import time -from flask import Flask, jsonify, render_template, request, redirect, url_for, session, flash -from sqlalchemy import insert -from app import app, db, jellyfin, functions, device_id,sp -from app.models import Playlist,Track, playlist_tracks -from spotipy.exceptions import SpotifyException - - -from jellyfin.objects import PlaylistMetadata - - - -@app.route('/jellyfin_playlists') -@functions.jellyfin_login_required -def jellyfin_playlists(): - try: - # Fetch playlists from Jellyfin - playlists = jellyfin.get_playlists(session_token=functions._get_token_from_sessioncookie()) - spotify_data = {'playlists': {'items': []}} - - # Extract Spotify playlist IDs from the database - for pl in playlists: - # Retrieve the playlist from the database using Jellyfin ID - from_db = Playlist.query.filter_by(jellyfin_id=pl['Id']).first() - playlist_data = None - not_found = False - if from_db and from_db.provider_playlist_id: - pl_id = from_db.provider_playlist_id - try: - playlist_data = functions.get_cached_spotify_playlist(pl_id) - - except SpotifyException as e: - app.logger.error(f"Error Fetching Playlist {pl_id}: {e}") - not_found = 'http status: 404' in str(e) - if not_found: - playlist_data = { - 'status':'red', - 'description': 'Playlist has most likely been removed. You can keep it, but won´t receive Updates.', - 'id': from_db.provider_playlist_id, - 'name' : from_db.name - - } - - if playlist_data: - spotify_data['playlists']['items'].append(playlist_data) - - else: - app.logger.warning(f"No database entry found for Jellyfin playlist ID: {pl['Id']}") - - prepared_data = functions.prepPlaylistData(spotify_data) - - return render_template('jellyfin_playlists.html', playlists=prepared_data) - except SpotifyException as e: - app.logger.error(f"Error fetching monitored playlists: {e}") - error_data, error_message = e, f'Could not retrieve monitored Playlists. Please try again later. This is most likely due to an Error in the Spotify API or an rate limit has been reached.' - return render_template('jellyfin_playlists.html', playlists=functions.prepPlaylistData({'playlists': {'items': []}}), error_message=error_message,error_data = error_data) - - except Exception as e: - app.logger.error(f"Error in /jellyfin_playlists route: {str(e)}") - flash('An error occurred while fetching playlists.', 'danger') - return render_template('jellyfin_playlists.html', playlists=functions.prepPlaylistData({'playlists': {'items': []}}), error_message='An error occurred while fetching playlists.',error_data = e) - - -@app.route('/addplaylist', methods=['POST']) -@functions.jellyfin_login_required -def add_playlist(): - playlist_id = request.form.get('item_id') # HTMX sends the form data - playlist_name = request.form.get('item_name') # Optionally retrieve playlist name from the form - if not playlist_id: - flash('No playlist ID provided') - return '' - - try: - # Fetch playlist from Spotify API (or any relevant API) - playlist_data = functions.get_cached_spotify_playlist(playlist_id) - - # Check if playlist already exists in the database - playlist = Playlist.query.filter_by(provider_playlist_id=playlist_id).first() - - if not playlist: - # Add new playlist if it doesn't exist - # create the playlist via api key, with the first admin as 'owner' - fromJellyfin = jellyfin.create_music_playlist(functions._get_api_token(),playlist_data['name'],[],functions._get_admin_id())['Id'] - playlist = Playlist(name=playlist_data['name'], provider_playlist_id=playlist_id,provider_uri=playlist_data['uri'],track_count = playlist_data['tracks']['total'], tracks_available=0, jellyfin_id = fromJellyfin) - db.session.add(playlist) - db.session.commit() - if app.config['START_DOWNLOAD_AFTER_PLAYLIST_ADD']: - functions.manage_task('download_missing_tracks') - - - # Get the logged-in user - user = functions._get_logged_in_user() - playlist.tracks_available = 0 - - spotify_tracks = {} - offset = 0 - while True: - playlist_items = sp.playlist_items(playlist.provider_playlist_id, offset=offset, limit=100) - items = playlist_items['items'] - spotify_tracks.update({offset + idx: track['track'] for idx, track in enumerate(items) if track['track']}) - - if len(items) < 100: # No more tracks to fetch - break - offset += 100 # Move to the next batch - for idx, track_data in spotify_tracks.items(): - track_info = track_data - if not track_info: - continue - track = Track.query.filter_by(provider_track_id=track_info['id']).first() - - if not track: - # Add new track if it doesn't exist - track = Track(name=track_info['name'], provider_track_id=track_info['id'], provider_uri=track_info['uri'], downloaded=False) - db.session.add(track) - db.session.commit() - elif track.downloaded: - playlist.tracks_available += 1 - db.session.commit() - - # Add track to playlist with order if it's not already associated - if track not in playlist.tracks: - # Insert into playlist_tracks with track order - stmt = insert(playlist_tracks).values( - playlist_id=playlist.id, - track_id=track.id, - track_order=idx # Maintain the order of tracks - ) - db.session.execute(stmt) - db.session.commit() - - functions.update_playlist_metadata(playlist,playlist_data) - - if playlist not in user.playlists: - user.playlists.append(playlist) - db.session.commit() - jellyfin.add_users_to_playlist(session_token=functions._get_api_token(), user_id=functions._get_admin_id(),playlist_id = playlist.jellyfin_id,user_ids= [user.jellyfin_user_id]) - flash(f'Playlist "{playlist_data["name"]}" successfully added','success') - - else: - flash(f'Playlist "{playlist_data["name"]}" already in your list') - item = { - "name" : playlist_data["name"], - "id" : playlist_id, - "can_add":False, - "can_remove":True, - "jellyfin_id" : playlist.jellyfin_id - } - return render_template('partials/_add_remove_button.html',item= item) - - - - - except Exception as e: - flash(str(e)) - return '' - - -@app.route('/delete_playlist/', methods=['DELETE']) -@functions.jellyfin_login_required -def delete_playlist(playlist_id): - # Logic to delete the playlist using JellyfinClient - try: - user = functions._get_logged_in_user() - for pl in user.playlists: - if pl.jellyfin_id == playlist_id: - user.playlists.remove(pl) - playlist = pl - jellyfin.remove_user_from_playlist(session_token= functions._get_api_token(), playlist_id= playlist_id, user_id=user.jellyfin_user_id) - db.session.commit() - flash('Playlist removed') - item = { - "name" : playlist.name, - "id" : playlist.provider_playlist_id, - "can_add":True, - "can_remove":False, - "jellyfin_id" : playlist.jellyfin_id - } - return render_template('partials/_add_remove_button.html',item= item) - except Exception as e: - flash(f'Failed to remove item: {str(e)}') - - -@app.route('/wipe_playlist/', methods=['DELETE']) -@functions.jellyfin_admin_required -def wipe_playlist(playlist_id): - playlist = Playlist.query.filter_by(jellyfin_id=playlist_id).first() - name = "" - id = "" - jf_id = "" - try: - jellyfin.remove_item(session_token=functions._get_api_token(), playlist_id=playlist_id) - except Exception as e: - flash(f"Jellyfin API Error: {str(e)}") - if playlist: - # Delete the playlist - name = playlist.name - id = playlist.provider_playlist_id - jf_id = playlist.jellyfin_id - db.session.delete(playlist) - db.session.commit() - flash('Playlist Deleted', category='info') - item = { - "name" : name, - "id" : id, - "can_add":True, - "can_remove":False, - "jellyfin_id" : jf_id - } - return render_template('partials/_add_remove_button.html',item= item) - -@functions.jellyfin_login_required -@app.route('/get_jellyfin_stream/') -def get_jellyfin_stream(jellyfin_id): - user_id = session['jellyfin_user_id'] # Beispiel: dynamischer Benutzer - api_key = functions._get_token_from_sessioncookie() # Beispiel: dynamischer API-Schlüssel - stream_url = f"{app.config['JELLYFIN_SERVER_URL']}/Audio/{jellyfin_id}/universal?UserId={user_id}&DeviceId={device_id}&MaxStreamingBitrate=140000000&Container=opus,webm|opus,mp3,aac,m4a|aac,m4b|aac,flac,webma,webm|webma,wav,ogg&TranscodingContainer=mp4&TranscodingProtocol=hls&AudioCodec=aac&api_key={api_key}&PlaySessionId={int(time.time())}&StartTimeTicks=0&EnableRedirection=true&EnableRemoteMedia=false" - return jsonify({'stream_url': stream_url}) - -@app.route('/search_jellyfin', methods=['GET']) -@functions.jellyfin_login_required -def search_jellyfin(): - search_query = request.args.get('search_query') - spotify_id = request.args.get('spotify_id') - if search_query: - results = jellyfin.search_music_tracks(functions._get_token_from_sessioncookie(), search_query) - # Render only the search results section as response - return render_template('partials/_jf_search_results.html', results=results,spotify_id= spotify_id,search_query = search_query) - return jsonify({'error': 'No search query provided'}), 400 \ No newline at end of file diff --git a/app/routes.py b/app/routes.py deleted file mode 100644 index fcf0cb1..0000000 --- a/app/routes.py +++ /dev/null @@ -1,355 +0,0 @@ -import json -import re -from flask import Flask, Response, jsonify, render_template, request, redirect, url_for, session, flash, Blueprint, g -from app import app, db, functions, sp, jellyfin, celery, jellyfin_admin_token, jellyfin_admin_id,device_id, cache, read_dev_build_file, tasks -from app.models import JellyfinUser,Playlist,Track -from celery.result import AsyncResult - -from app.providers import base -from app.providers.base import MusicProviderClient -from app.providers.spotify import SpotifyClient -from app.registry.music_provider_registry import MusicProviderRegistry -from .version import __version__ -from spotipy.exceptions import SpotifyException - -pl_bp = Blueprint('playlist', __name__) -@pl_bp.before_request -def set_active_provider(): - """ - Middleware to select the active provider based on request parameters. - """ - provider_id = request.args.get('provider', 'Spotify') # Default to Spotify - try: - g.music_provider = MusicProviderRegistry.get_provider(provider_id) - except ValueError as e: - return {"error": str(e)}, 400 - -@app.context_processor -def add_context(): - unlinked_track_count = len(Track.query.filter_by(downloaded=True,jellyfin_id=None).all()) - version = f"v{__version__}{read_dev_build_file()}" - return dict(unlinked_track_count = unlinked_track_count, version = version, config = app.config) - - -# this feels wrong -skip_endpoints = ['task_status'] -@app.after_request -def render_messages(response: Response) -> Response: - if request.headers.get("HX-Request"): - if request.endpoint not in skip_endpoints: - messages = render_template("partials/alerts.jinja2") - response.headers['HX-Trigger'] = 'showToastMessages' - response.data = response.data + messages.encode("utf-8") - return response - - - -@app.route('/admin/tasks') -@functions.jellyfin_admin_required -def task_manager(): - statuses = {} - for task_name, task_id in functions.TASK_STATUS.items(): - if task_id: - result = AsyncResult(task_id) - statuses[task_name] = {'state': result.state, 'info': result.info if result.info else {}} - else: - statuses[task_name] = {'state': 'NOT STARTED', 'info': {}} - - return render_template('admin/tasks.html', tasks=statuses,lock_keys = functions.LOCK_KEYS) - -@app.route('/admin') -@app.route('/admin/link_issues') -@functions.jellyfin_admin_required -def link_issues(): - unlinked_tracks = Track.query.filter_by(downloaded=True,jellyfin_id=None).all() - tracks = [] - for ult in unlinked_tracks: - sp_track = functions.get_cached_spotify_track(ult.provider_track_id) - duration_ms = sp_track['duration_ms'] - minutes = duration_ms // 60000 - seconds = (duration_ms % 60000) // 1000 - tracks.append({ - 'title': sp_track['name'], - 'artist': ', '.join([artist['name'] for artist in sp_track['artists']]), - 'url': sp_track['external_urls']['spotify'], - 'duration': f'{minutes}:{seconds:02d}', - 'preview_url': sp_track['preview_url'], - 'downloaded': ult.downloaded, - 'filesystem_path': ult.filesystem_path, - 'jellyfin_id': ult.jellyfin_id, - 'spotify_id': sp_track['id'], - 'duration_ms': duration_ms, - 'download_status' : ult.download_status - }) - - return render_template('admin/link_issues.html' , tracks = tracks ) - - - -@app.route('/run_task/', methods=['POST']) -@functions.jellyfin_admin_required -def run_task(task_name): - status, info = functions.manage_task(task_name) - - # Rendere nur die aktualisierte Zeile der Task - task_info = {task_name: {'state': status, 'info': info}} - return render_template('partials/_task_status.html', tasks=task_info) - - -@app.route('/task_status') -@functions.jellyfin_admin_required -def task_status(): - statuses = {} - for task_name, task_id in functions.TASK_STATUS.items(): - if task_id: - result = AsyncResult(task_id) - statuses[task_name] = {'state': result.state, 'info': result.info if result.info else {}} - else: - statuses[task_name] = {'state': 'NOT STARTED', 'info': {}} - - # Render the HTML partial template instead of returning JSON - return render_template('partials/_task_status.html', tasks=statuses) - - - -@app.route('/') -@functions.jellyfin_login_required -def index(): - users = JellyfinUser.query.all() - return render_template('index.html', user=session['jellyfin_user_name'], users=users) - -@app.route('/login', methods=['GET', 'POST']) -def login(): - if request.method == 'POST': - username = request.form['username'] - password = request.form['password'] - try: - jellylogin = jellyfin.login_with_password(username=username, password=password) - if jellylogin: - session['jellyfin_access_token'], session['jellyfin_user_id'], session['jellyfin_user_name'],session['is_admin'] = jellylogin - session['debug'] = app.debug - # Check if the user already exists - user = JellyfinUser.query.filter_by(jellyfin_user_id=session['jellyfin_user_id']).first() - if not user: - # Add the user to the database if they don't exist - new_user = JellyfinUser(name=session['jellyfin_user_name'], jellyfin_user_id=session['jellyfin_user_id'], is_admin = session['is_admin']) - db.session.add(new_user) - db.session.commit() - - return redirect('/') - except: - flash('Login failed. Please check your Jellyfin credentials and try again.', 'error') - return redirect(url_for('login')) - - return render_template('login.html') - -@app.route('/logout') -def logout(): - session.pop('jellyfin_user_name', None) - session.pop('jellyfin_access_token', None) - return redirect(url_for('login')) - -@app.route('/add_single',methods=['GET']) -@functions.jellyfin_login_required -def add_single(): - playlist = request.args.get('playlist') - error = None - errdata= None - if playlist: - parsed = sp._get_id(type='playlist',id=playlist) - if parsed: - try: - functions.get_cached_spotify_playlist(parsed) - - return redirect(f'/playlist/view/{parsed}') - except SpotifyException as e: - url_match = re.search(sp._regex_spotify_url, playlist) - if url_match is not None: - resp = functions.fetch_spotify_playlist(playlist,None) - parsed_data = functions.parse_spotify_playlist_html(resp) - error = (f'Playlist can´t be fetched') - - errdata = str(e) - - return render_template('index.html',error_message = error, error_data = errdata) - - - - -@app.route('/playlists') -@app.route('/categories') -@app.route('/playlists/monitored') -@functions.jellyfin_login_required -def loaditems(): - country = app.config['SPOTIFY_COUNTRY_CODE'] - offset = int(request.args.get('offset', 0)) # Get the offset (default to 0 for initial load) - limit = 20 # Define a limit for pagination - additional_query = '' - items_subtitle = '' - error_message = None # Placeholder for error messages - error_data = '' - if request.path == '/playlists/monitored': - try: - db_playlists = db.session.query(Playlist).offset(offset).limit(limit).all() - max_items = db.session.query(Playlist).count() - - provider_playlist_ids = [playlist.provider_playlist_id for playlist in db_playlists] - spotify_data = functions.get_cached_spotify_playlists(tuple(provider_playlist_ids)) - for x in spotify_data['playlists']['items']: - for from_db in db_playlists: - if x['id'] == from_db.provider_playlist_id: - x['name'] = from_db.name - data = functions.prepPlaylistData(spotify_data) - items_title = "Monitored Playlists" - items_subtitle = "These playlists are already monitored by the Server. If you add one to your Jellyfin account, they will be available immediately." - except SpotifyException as e: - app.logger.error(f"Error fetching monitored playlists: {e}") - data, max_items, items_title = [], e, f'Could not retrieve monitored Playlists. Please try again later. This is most likely due to an Error in the Spotify API or an rate limit has been reached.' - error_message = items_title - error_data = max_items - - elif request.path == '/playlists': - cat = request.args.get('cat', None) - if cat is not None: - data, max_items, items_title = functions.getCategoryPlaylists(category=cat, offset=offset) - if not data: # Check if data is empty - error_message = items_title # Set the error message from the function - error_data = max_items - additional_query += f"&cat={cat}" - else: - data, max_items, items_title = functions.getFeaturedPlaylists(country=country, offset=offset) - if not data: # Check if data is empty - error_message = items_title # Set the error message from the function - error_data = max_items - - elif request.path == '/categories': - try: - data, max_items, items_title = functions.getCategories(country=country, offset=offset) - except Exception as e: - app.logger.error(f"Error fetching categories: {e}") - data, max_items, items_title = [], e, f'Error: Could not load categories. Please try again later. ' - error_message = items_title - error_data = max_items - - - next_offset = offset + len(data) - total_items = max_items - context = { - 'items': data, - 'next_offset': next_offset, - 'total_items': total_items, - 'endpoint': request.path, - 'items_title': items_title, - 'items_subtitle': items_subtitle, - 'additional_query': additional_query, - 'error_message': error_message, # Pass error message to the template - 'error_data': error_data, # Pass error message to the template - } - - if request.headers.get('HX-Request'): # Check if the request is from HTMX - return render_template('partials/_spotify_items.html', **context) - else: - return render_template('items.html', **context) - - -@app.route('/search') -@functions.jellyfin_login_required -def searchResults(): - query = request.args.get('query') - context = {} - if query: - # Add your logic here to perform the search on Spotify (or Jellyfin) - search_result = sp.search(q = query, type= 'playlist',limit= 50, market=app.config['SPOTIFY_COUNTRY_CODE']) - context = { - #'artists' : functions.prepArtistData(search_result ), - 'playlists' : functions.prepPlaylistData(search_result ), - #'albums' : functions.prepAlbumData(search_result ), - 'query' : query - } - return render_template('search.html', **context) - else: - return render_template('search.html', query=None, results={}) - - -@pl_bp.route('/playlist/view/') -@functions.jellyfin_login_required -def get_playlist_tracks(playlist_id): - provider: MusicProviderClient = g.music_provider # Explicit type hint for g.music_provider - playlist: base.Playlist = provider.get_playlist(playlist_id) - tracks = functions.get_tracks_for_playlist(playlist.tracks) # Deine Funktion, um Tracks zu holen - # Berechne die gesamte Dauer der Playlist - total_duration_ms = sum([track['track']['duration_ms'] for track in data['tracks']['items'] if track['track']]) - - # Konvertiere die Gesamtdauer in ein lesbares Format - hours, remainder = divmod(total_duration_ms // 1000, 3600) - minutes, seconds = divmod(remainder, 60) - - # Formatierung der Dauer - if hours > 0: - total_duration = f"{hours}h {minutes}min" - else: - total_duration = f"{minutes}min" - - return render_template( - 'tracks_table.html', - tracks=tracks, - total_duration=total_duration, - track_count=len(data['tracks']), - playlist_name=data['name'], - playlist_cover=data['images'][0]['url'], - playlist_description=data['description'], - last_updated = data['prepped_data'][0]['last_updated'], - last_changed = data['prepped_data'][0]['last_changed'], - item = data['prepped_data'][0], - - ) -@app.route('/associate_track', methods=['POST']) -@functions.jellyfin_login_required -def associate_track(): - jellyfin_id = request.form.get('jellyfin_id') - spotify_id = request.form.get('spotify_id') - - if not jellyfin_id or not spotify_id: - flash('Missing Jellyfin or Spotify ID') - - # Retrieve the track by Spotify ID - track = Track.query.filter_by(provider_track_id=spotify_id).first() - - if not track: - flash('Track not found') - return '' - - # Associate the Jellyfin ID with the track - track.jellyfin_id = jellyfin_id - - try: - # Commit the changes to the database - db.session.commit() - flash("Track associated","success") - return '' - except Exception as e: - db.session.rollback() # Roll back the session in case of an error - flash(str(e)) - return '' - - -@app.route("/unlock_key",methods = ['POST']) -@functions.jellyfin_admin_required -def unlock_key(): - - key_name = request.form.get('inputLockKey') - if key_name: - tasks.release_lock(key_name) - flash(f'Lock {key_name} released', category='success') - return '' - - -@app.route('/test') -def test(): - playlist_id = "37i9dQZF1DX12qgyzUprB6" - client = SpotifyClient(cookie_file='/jellyplist/open.spotify.com_cookies.txt') - client.authenticate() - pl = client.get_playlist(playlist_id=playlist_id) - browse = client.browse_all() - page = client.browse_page(browse[0].items[12]) - return '' \ No newline at end of file diff --git a/templates/partials/_spotify_item.html b/templates/partials/_spotify_item.html deleted file mode 100644 index 0ffd45d..0000000 --- a/templates/partials/_spotify_item.html +++ /dev/null @@ -1,49 +0,0 @@ -
-
- - - {% if item.status %} - - {% if item.track_count > 0 %} - {{ item.tracks_available }} / {{ item.tracks_linked}} / {{ item.track_count}} - {% else %} - not Available - {% endif %} - - {% endif %} - - -
- {{ item.name }} -
- - -
-
-
{{ item.name }}
-

{{ item.description }}

-
-
- {% if item.type == 'category'%} - - - - {%else%} - - - - - {%endif%} - {% include 'partials/_add_remove_button.html' %} -
-
- -
-
\ No newline at end of file diff --git a/templates/partials/_spotify_items.html b/templates/partials/_spotify_items.html deleted file mode 100644 index 896008b..0000000 --- a/templates/partials/_spotify_items.html +++ /dev/null @@ -1,24 +0,0 @@ -{% for item in items %} - {% include 'partials/_spotify_item.html' %} - -{% endfor %} - -{% if next_offset < total_items %} -
- Loading more items... -
-{% endif %} - - \ No newline at end of file diff --git a/templates/partials/_toast.html b/templates/partials/_toast.html deleted file mode 100644 index fa4e494..0000000 --- a/templates/partials/_toast.html +++ /dev/null @@ -1,18 +0,0 @@ - - - \ No newline at end of file From 6b78b90ee7c657d822f1505fa9257a4eec4a14f5 Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:35:23 +0000 Subject: [PATCH 22/42] feature: add possibility to directly add a playlist from the / page --- static/css/styles.css | 45 ++++++++++++++++++++++++++++++++++++++++++- templates/index.html | 31 +++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/static/css/styles.css b/static/css/styles.css index faedd5a..3c1e918 100644 --- a/static/css/styles.css +++ b/static/css/styles.css @@ -57,4 +57,47 @@ body { .modal-dialog { max-width: 90%; /* New width for default modal */ } -} \ No newline at end of file +} +.searchbar{ + margin-bottom: auto; + margin-top: auto; + height: 60px; + background-color: #353b48; + border-radius: 30px; + padding: 10px; + } + + .search_input{ + color: white; + border: 0; + outline: 0; + background: none; + width: 450px; + caret-color:transparent; + line-height: 40px; + transition: width 0.4s linear; + } + + .searchbar:hover > .search_input{ + /* padding: 0 10px; */ + width: 450px; + caret-color:red; + /* transition: width 0.4s linear; */ + } + + .searchbar:hover > .search_icon{ + background: white; + color: #e74c3c; + } + + .search_icon{ + height: 40px; + width: 40px; + float: right; + display: flex; + justify-content: center; + align-items: center; + border-radius: 50%; + color:white; + text-decoration:none; + } \ No newline at end of file diff --git a/templates/index.html b/templates/index.html index a6a944c..db894fe 100644 --- a/templates/index.html +++ b/templates/index.html @@ -2,5 +2,36 @@ {% block content %} +
+
+
+ + + + + + {% if error_message %} + +{% endif %} +
+
+ +
+ {% endblock %} From d5aee793a03f8105144d7a7f75b710d6956e923b Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:35:45 +0000 Subject: [PATCH 23/42] updated .gitignore to not include any cookies at all --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8f96a38..c07fcde 100644 --- a/.gitignore +++ b/.gitignore @@ -73,7 +73,7 @@ coverage/ # macOS .DS_Store .cache -cookies*.txt +*cookies*.txt *.code-workspace set_env.sh notes.md \ No newline at end of file From 3a26c054a0b8867034b52788bac303580d9903a5 Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:38:47 +0000 Subject: [PATCH 24/42] added blueprint and restructured existing routes --- app/__init__.py | 7 +- app/routes/__init__.py | 17 ++ app/routes/jellyfin_routes.py | 202 +++++++++++++++++ app/routes/routes.py | 412 ++++++++++++++++++++++++++++++++++ 4 files changed, 635 insertions(+), 3 deletions(-) create mode 100644 app/routes/__init__.py create mode 100644 app/routes/jellyfin_routes.py create mode 100644 app/routes/routes.py diff --git a/app/__init__.py b/app/__init__.py index 841606d..12e751e 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -161,10 +161,11 @@ celery.set_default() app.logger.info(f'Jellyplist {__version__}{read_dev_build_file()} started') app.logger.debug(f"Debug logging active") -from app import routes -app.register_blueprint(routes.pl_bp) -from app import jellyfin_routes, tasks +from app.routes import pl_bp, routes, jellyfin_routes +app.register_blueprint(pl_bp) + +from . import tasks if "worker" in sys.argv: tasks.release_lock("download_missing_tracks_lock") diff --git a/app/routes/__init__.py b/app/routes/__init__.py new file mode 100644 index 0000000..055d915 --- /dev/null +++ b/app/routes/__init__.py @@ -0,0 +1,17 @@ +from flask import Blueprint, request, g +from app import app +from app.registry.music_provider_registry import MusicProviderRegistry + +pl_bp = Blueprint('playlist', __name__) + +@pl_bp.before_request +def set_active_provider(): + """ + Middleware to select the active provider based on request parameters. + """ + app.logger.debug(f"Setting active provider: {request.args.get('provider', 'Spotify')}") + provider_id = request.args.get('provider', 'Spotify') # Default to Spotify + try: + g.music_provider = MusicProviderRegistry.get_provider(provider_id) + except ValueError as e: + return {"error": str(e)}, 400 \ No newline at end of file diff --git a/app/routes/jellyfin_routes.py b/app/routes/jellyfin_routes.py new file mode 100644 index 0000000..a781440 --- /dev/null +++ b/app/routes/jellyfin_routes.py @@ -0,0 +1,202 @@ +from collections import defaultdict +import time +from flask import Blueprint, Flask, jsonify, render_template, request, redirect, url_for, session, flash +from sqlalchemy import insert +from app import app, db, jellyfin, functions, device_id,sp +from app.models import JellyfinUser, Playlist,Track, playlist_tracks +from spotipy.exceptions import SpotifyException + + +from app.registry.music_provider_registry import MusicProviderRegistry +from jellyfin.objects import PlaylistMetadata +from app.routes import pl_bp + +@app.route('/jellyfin_playlists') +@functions.jellyfin_login_required +def jellyfin_playlists(): + playlists = jellyfin.get_playlists(session_token=functions._get_token_from_sessioncookie()) + playlists_by_provider = defaultdict(list) + provider_playlists_data = {} + + for pl in playlists: + from_db : Playlist | None = Playlist.query.filter_by(jellyfin_id=pl['Id']).first() + if from_db and from_db.provider_playlist_id: + pl_id = from_db.provider_playlist_id + playlists_by_provider[from_db.provider_id].append(from_db) + + # 3. Fetch all Data from the provider using the get_playlist() method + for provider_id, playlists in playlists_by_provider.items(): + try: + provider_client = MusicProviderRegistry.get_provider(provider_id) + except ValueError: + flash(f"Provider {provider_id} not found.", "error") + continue + + combined_playlists = [] + for pl in playlists: + provider_playlist = provider_client.get_playlist(pl.provider_playlist_id) + # 4. Convert the playlists to CombinedPlaylistData + combined_data = functions.prepPlaylistData(provider_playlist) + if combined_data: + combined_playlists.append(combined_data) + + provider_playlists_data[provider_id] = combined_playlists + + # 5. Display the resulting Groups in a template called 'monitored_playlists.html', one Heading per Provider + return render_template('monitored_playlists.html', provider_playlists_data=provider_playlists_data,title="Jellyfin Playlists" , subtitle="Playlists you have added to Jellyfin") + +@pl_bp.route('/addplaylist', methods=['POST']) +@functions.jellyfin_login_required +def add_playlist(): + playlist_id = request.form.get('item_id') + playlist_name = request.form.get('item_name') + # also get the provider id from the query params + provider_id = request.args.get('provider') + if not playlist_id: + flash('No playlist ID provided') + return '' + # if no provider_id is provided, then show an error and return an empty string + if not provider_id: + flash('No provider ID provided') + return '' + try: + # get the playlist from the correct provider + provider_client = MusicProviderRegistry.get_provider(provider_id) + playlist_data = provider_client.get_playlist(playlist_id) + # Check if playlist already exists in the database, using the provider_id and the provider_playlist_id + playlist = Playlist.query.filter_by(provider_playlist_id=playlist_id, provider_id=provider_id).first() + # Add new playlist in the database if it doesn't exist + # create the playlist via api key, with the first admin as 'owner' + if not playlist: + fromJellyfin = jellyfin.create_music_playlist(functions._get_api_token(),playlist_data.name,[],functions._get_admin_id())['Id'] + playlist = Playlist(name=playlist_data.name, provider_playlist_id=playlist_id,provider_uri=playlist_data.uri,track_count = len(playlist_data.tracks), tracks_available=0, jellyfin_id = fromJellyfin, provider_id=provider_id) + db.session.add(playlist) + db.session.commit() + if app.config['START_DOWNLOAD_AFTER_PLAYLIST_ADD']: + functions.manage_task('download_missing_tracks') + # Get the logged-in user + user : JellyfinUser = functions._get_logged_in_user() + playlist.tracks_available = 0 + + for idx, track_data in enumerate(playlist_data.tracks): + + track = Track.query.filter_by(provider_track_id=track_data.track.id, provider_id=provider_id).first() + + if not track: + # Add new track if it doesn't exist + track = Track(name=track_data.track.name, provider_track_id=track_data.track.id, provider_uri=track_data.track.uri, downloaded=False,provider_id = provider_id) + db.session.add(track) + db.session.commit() + elif track.downloaded: + playlist.tracks_available += 1 + db.session.commit() + + # Add track to playlist with order if it's not already associated + if track not in playlist.tracks: + # Insert into playlist_tracks with track order + stmt = insert(playlist_tracks).values( + playlist_id=playlist.id, + track_id=track.id, + track_order=idx # Maintain the order of tracks + ) + db.session.execute(stmt) + db.session.commit() + + functions.update_playlist_metadata(playlist,playlist_data) + + if playlist not in user.playlists: + user.playlists.append(playlist) + db.session.commit() + jellyfin.add_users_to_playlist(session_token=functions._get_api_token(), user_id=functions._get_admin_id(),playlist_id = playlist.jellyfin_id,user_ids= [user.jellyfin_user_id]) + flash(f'Playlist "{playlist_data.name}" successfully added','success') + + else: + flash(f'Playlist "{playlist_data.name}" already in your list') + item = { + "name" : playlist_data.name, + "id" : playlist_id, + "can_add":False, + "can_remove":True, + "jellyfin_id" : playlist.jellyfin_id + } + return render_template('partials/_add_remove_button.html',item= item) + + + + + except Exception as e: + flash(str(e)) + return '' + + +@app.route('/delete_playlist/', methods=['DELETE']) +@functions.jellyfin_login_required +def delete_playlist(playlist_id): + # Logic to delete the playlist using JellyfinClient + try: + user = functions._get_logged_in_user() + for pl in user.playlists: + if pl.jellyfin_id == playlist_id: + user.playlists.remove(pl) + playlist = pl + jellyfin.remove_user_from_playlist(session_token= functions._get_api_token(), playlist_id= playlist_id, user_id=user.jellyfin_user_id) + db.session.commit() + flash('Playlist removed') + item = { + "name" : playlist.name, + "id" : playlist.provider_playlist_id, + "can_add":True, + "can_remove":False, + "jellyfin_id" : playlist.jellyfin_id + } + return render_template('partials/_add_remove_button.html',item= item) + except Exception as e: + flash(f'Failed to remove item: {str(e)}') + + +@app.route('/wipe_playlist/', methods=['DELETE']) +@functions.jellyfin_admin_required +def wipe_playlist(playlist_id): + playlist = Playlist.query.filter_by(jellyfin_id=playlist_id).first() + name = "" + id = "" + jf_id = "" + try: + jellyfin.remove_item(session_token=functions._get_api_token(), playlist_id=playlist_id) + except Exception as e: + flash(f"Jellyfin API Error: {str(e)}") + if playlist: + # Delete the playlist + name = playlist.name + id = playlist.provider_playlist_id + jf_id = playlist.jellyfin_id + db.session.delete(playlist) + db.session.commit() + flash('Playlist Deleted', category='info') + item = { + "name" : name, + "id" : id, + "can_add":True, + "can_remove":False, + "jellyfin_id" : jf_id + } + return render_template('partials/_add_remove_button.html',item= item) + +@functions.jellyfin_login_required +@app.route('/get_jellyfin_stream/') +def get_jellyfin_stream(jellyfin_id): + user_id = session['jellyfin_user_id'] # Beispiel: dynamischer Benutzer + api_key = functions._get_token_from_sessioncookie() # Beispiel: dynamischer API-Schlüssel + stream_url = f"{app.config['JELLYFIN_SERVER_URL']}/Audio/{jellyfin_id}/universal?UserId={user_id}&DeviceId={device_id}&MaxStreamingBitrate=140000000&Container=opus,webm|opus,mp3,aac,m4a|aac,m4b|aac,flac,webma,webm|webma,wav,ogg&TranscodingContainer=mp4&TranscodingProtocol=hls&AudioCodec=aac&api_key={api_key}&PlaySessionId={int(time.time())}&StartTimeTicks=0&EnableRedirection=true&EnableRemoteMedia=false" + return jsonify({'stream_url': stream_url}) + +@app.route('/search_jellyfin', methods=['GET']) +@functions.jellyfin_login_required +def search_jellyfin(): + search_query = request.args.get('search_query') + provider_track_id = request.args.get('provider_track_id') + if search_query: + results = jellyfin.search_music_tracks(functions._get_token_from_sessioncookie(), search_query) + # Render only the search results section as response + return render_template('partials/_jf_search_results.html', results=results,provider_track_id= provider_track_id,search_query = search_query) + return jsonify({'error': 'No search query provided'}), 400 \ No newline at end of file diff --git a/app/routes/routes.py b/app/routes/routes.py new file mode 100644 index 0000000..0ee8e3b --- /dev/null +++ b/app/routes/routes.py @@ -0,0 +1,412 @@ +import json +import os +import re +from flask import Flask, Response, jsonify, render_template, request, redirect, url_for, session, flash, Blueprint, g +from app import app, db, functions, sp, jellyfin, celery, jellyfin_admin_token, jellyfin_admin_id,device_id, cache, read_dev_build_file, tasks +from app.classes import AudioProfile, CombinedPlaylistData +from app.models import JellyfinUser,Playlist,Track +from celery.result import AsyncResult +from typing import List + +from app.providers import base +from app.providers.base import MusicProviderClient +from app.providers.spotify import SpotifyClient +from app.registry.music_provider_registry import MusicProviderRegistry +from ..version import __version__ +from spotipy.exceptions import SpotifyException +from collections import defaultdict +from app.routes import pl_bp + + +@app.context_processor +def add_context(): + unlinked_track_count = len(Track.query.filter_by(downloaded=True,jellyfin_id=None).all()) + version = f"v{__version__}{read_dev_build_file()}" + return dict(unlinked_track_count = unlinked_track_count, version = version, config = app.config , registered_providers = MusicProviderRegistry.list_providers()) + + +# this feels wrong +skip_endpoints = ['task_status'] +@app.after_request +def render_messages(response: Response) -> Response: + if request.headers.get("HX-Request"): + if request.endpoint not in skip_endpoints: + messages = render_template("partials/alerts.jinja2") + response.headers['HX-Trigger'] = 'showToastMessages' + response.data = response.data + messages.encode("utf-8") + return response + + + +@app.route('/admin/tasks') +@functions.jellyfin_admin_required +def task_manager(): + statuses = {} + for task_name, task_id in functions.TASK_STATUS.items(): + if task_id: + result = AsyncResult(task_id) + statuses[task_name] = {'state': result.state, 'info': result.info if result.info else {}} + else: + statuses[task_name] = {'state': 'NOT STARTED', 'info': {}} + + return render_template('admin/tasks.html', tasks=statuses,lock_keys = functions.LOCK_KEYS) + +@app.route('/admin') +@app.route('/admin/link_issues') +@functions.jellyfin_admin_required +def link_issues(): + # add the ability to pass a query parameter to dislplay even undownloaded tracks + list_undownloaded = request.args.get('list_undownloaded') + if list_undownloaded: + unlinked_tracks = Track.query.filter_by(jellyfin_id=None).all() + else: + unlinked_tracks = Track.query.filter_by(downloaded=True,jellyfin_id=None).all() + tracks = [] + for ult in unlinked_tracks: + provider_track = functions.get_cached_provider_track(ult.provider_track_id, ult.provider_id) + duration_ms = provider_track.duration_ms + minutes = duration_ms // 60000 + seconds = (duration_ms % 60000) // 1000 + tracks.append({ + 'title': provider_track.name, + 'artist': ', '.join([artist.name for artist in provider_track.artists]), + 'url': provider_track.external_urls, + 'duration': f'{minutes}:{seconds:02d}', + 'preview_url': '', + 'downloaded': ult.downloaded, + 'filesystem_path': ult.filesystem_path, + 'jellyfin_id': ult.jellyfin_id, + 'provider_track_id': provider_track.id, + 'duration_ms': duration_ms, + 'download_status' : ult.download_status, + 'provider_id' : ult.provider_id + }) + + return render_template('admin/link_issues.html' , tracks = tracks ) + + + +@app.route('/run_task/', methods=['POST']) +@functions.jellyfin_admin_required +def run_task(task_name): + status, info = functions.manage_task(task_name) + + # Rendere nur die aktualisierte Zeile der Task + task_info = {task_name: {'state': status, 'info': info}} + return render_template('partials/_task_status.html', tasks=task_info) + + +@app.route('/task_status') +@functions.jellyfin_admin_required +def task_status(): + statuses = {} + for task_name, task_id in functions.TASK_STATUS.items(): + if task_id: + result = AsyncResult(task_id) + statuses[task_name] = {'state': result.state, 'info': result.info if result.info else {}} + else: + statuses[task_name] = {'state': 'NOT STARTED', 'info': {}} + + # Render the HTML partial template instead of returning JSON + return render_template('partials/_task_status.html', tasks=statuses) + + + +@app.route('/') +@functions.jellyfin_login_required +def index(): + users = JellyfinUser.query.all() + return render_template('index.html', user=session['jellyfin_user_name'], users=users) + +@app.route('/login', methods=['GET', 'POST']) +def login(): + if request.method == 'POST': + username = request.form['username'] + password = request.form['password'] + try: + jellylogin = jellyfin.login_with_password(username=username, password=password) + if jellylogin: + session['jellyfin_access_token'], session['jellyfin_user_id'], session['jellyfin_user_name'],session['is_admin'] = jellylogin + session['debug'] = app.debug + # Check if the user already exists + user = JellyfinUser.query.filter_by(jellyfin_user_id=session['jellyfin_user_id']).first() + if not user: + # Add the user to the database if they don't exist + new_user = JellyfinUser(name=session['jellyfin_user_name'], jellyfin_user_id=session['jellyfin_user_id'], is_admin = session['is_admin']) + db.session.add(new_user) + db.session.commit() + + return redirect('/') + except: + flash('Login failed. Please check your Jellyfin credentials and try again.', 'error') + return redirect(url_for('login')) + + return render_template('login.html') + +@app.route('/logout') +def logout(): + session.pop('jellyfin_user_name', None) + session.pop('jellyfin_access_token', None) + return redirect(url_for('login')) + +@app.route('/playlist/open',methods=['GET']) +@functions.jellyfin_login_required +def openPlaylist(): + playlist = request.args.get('playlist') + error = None + errdata= None + if playlist: + for provider_id in MusicProviderRegistry.list_providers(): + try: + provider_client = MusicProviderRegistry.get_provider(provider_id) + extracted_playlist_id = provider_client.extract_playlist_id(playlist) + provider_playlist = provider_client.get_playlist(extracted_playlist_id) + + combined_data = functions.prepPlaylistData(provider_playlist) + if combined_data: + # If the playlist is found, redirect to the playlist view, but also include the provider ID in the URL + return redirect(url_for('playlist.get_playlist_tracks', playlist_id=extracted_playlist_id, provider=provider_id)) + except Exception as e: + error = f"Error fetching playlist from {provider_id}: {str(e)}" + errdata = e + + return render_template('index.html',error_message = error, error_data = errdata) + +@pl_bp.route('/browse') +@functions.jellyfin_login_required +def browse(): + provider: MusicProviderClient = g.music_provider + + browse_data = provider.browse() + return render_template('browse.html', browse_data=browse_data,provider_id=provider._identifier) + +@pl_bp.route('/browse/page/') +@functions.jellyfin_login_required +def browse_page(page_id): + provider: MusicProviderClient = g.music_provider + combined_playlist_data : List[CombinedPlaylistData] = [] + + data = provider.browse_page(page_id) + for item in data: + cpd = functions.prepPlaylistData(item) + if cpd: + combined_playlist_data.append(cpd) + return render_template('browse_page.html', data=combined_playlist_data,provider_id=provider._identifier) + +@pl_bp.route('/playlists/monitored') +@functions.jellyfin_login_required +def monitored_playlists(): + + # 1. Get all Playlists from the Database. + all_playlists = Playlist.query.all() + + # 2. Group them by provider + playlists_by_provider = defaultdict(list) + for playlist in all_playlists: + playlists_by_provider[playlist.provider_id].append(playlist) + + provider_playlists_data = {} + # 3. Fetch all Data from the provider using the get_playlist() method + for provider_id, playlists in playlists_by_provider.items(): + try: + provider_client = MusicProviderRegistry.get_provider(provider_id) + except ValueError: + flash(f"Provider {provider_id} not found.", "error") + continue + + combined_playlists = [] + for pl in playlists: + provider_playlist = provider_client.get_playlist(pl.provider_playlist_id) + # 4. Convert the playlists to CombinedPlaylistData + combined_data = functions.prepPlaylistData(provider_playlist) + if combined_data: + combined_playlists.append(combined_data) + + provider_playlists_data[provider_id] = combined_playlists + + # 5. Display the resulting Groups in a template called 'monitored_playlists.html', one Heading per Provider + return render_template('monitored_playlists.html', provider_playlists_data=provider_playlists_data, title="Monitored Playlists", subtitle="Playlists which are already monitored by Jellyplist and are available immediately") + +@app.route('/search') +@functions.jellyfin_login_required +def searchResults(): + query = request.args.get('query') + context = {} + if query: + #iterate through every registered music provider and perform the search with it. + # Group the results by provider and display them using monitorerd_playlists.html + search_results = defaultdict(list) + for provider_id in MusicProviderRegistry.list_providers(): + try: + provider_client = MusicProviderRegistry.get_provider(provider_id) + results = provider_client.search_playlist(query) + for result in results: + search_results[provider_id].append(result) + except Exception as e: + flash(f"Error fetching search results from {provider_id}: {str(e)}", "error") + # the grouped search results, must be prepared using the prepPlaylistData function + for provider_id, playlists in search_results.items(): + combined_playlists = [] + for pl in playlists: + combined_data = functions.prepPlaylistData(pl) + if combined_data: + combined_playlists.append(combined_data) + search_results[provider_id] = combined_playlists + + context['provider_playlists_data'] = search_results + context['title'] = 'Search Results' + context['subtitle'] = 'Search results from all providers' + return render_template('monitored_playlists.html', **context) + +@pl_bp.route('/track_details/') +@functions.jellyfin_login_required +def track_details(track_id): + provider_id = request.args.get('provider') + if not provider_id: + return jsonify({'error': 'Provider not specified'}), 400 + + track = Track.query.filter_by(provider_track_id=track_id, provider_id=provider_id).first() + if not track: + return jsonify({'error': 'Track not found'}), 404 + + provider_track = functions.get_cached_provider_track(track.provider_track_id, track.provider_id) + # query also this track using the jellyfin id directly from jellyfin + if track.jellyfin_id: + jellyfin_track = jellyfin.get_item(session_token=functions._get_api_token(), item_id=track.jellyfin_id) + if jellyfin_track: + jellyfin_filesystem_path = jellyfin_track['Path'] + duration_ms = provider_track.duration_ms + minutes = duration_ms // 60000 + seconds = (duration_ms % 60000) // 1000 + + track_details = { + 'title': provider_track.name, + 'artist': ', '.join([artist.name for artist in provider_track.artists]), + 'url': provider_track.external_urls, + 'duration': f'{minutes}:{seconds:02d}', + 'downloaded': track.downloaded, + 'filesystem_path': track.filesystem_path, + 'jellyfin_id': track.jellyfin_id, + 'provider_track_id': provider_track.id, + 'provider_track_url': provider_track.external_urls[0].url if provider_track.external_urls else None, + 'duration_ms': duration_ms, + 'download_status': track.download_status, + 'provider_id': track.provider_id, + 'jellyfin_filesystem_path': jellyfin_filesystem_path if track.jellyfin_id else None, + } + + return render_template('partials/track_details.html', track=track_details) + +@pl_bp.route('/playlist/view/') +@functions.jellyfin_login_required +def get_playlist_tracks(playlist_id): + provider: MusicProviderClient = g.music_provider + playlist: base.Playlist = provider.get_playlist(playlist_id) + tracks = functions.get_tracks_for_playlist(playlist.tracks, provider_id=provider._identifier) + total_duration_ms = sum([track.duration_ms for track in tracks]) + + # Convert the total duration to a readable format + hours, remainder = divmod(total_duration_ms // 1000, 3600) + minutes, seconds = divmod(remainder, 60) + + # Format the duration + if hours > 0: + total_duration = f"{hours}h {minutes}min" + else: + total_duration = f"{minutes}min" + + return render_template( + 'tracks_table.html', + tracks=tracks, + total_duration=total_duration, + track_count=len(tracks), + provider_id = provider._identifier, + item=functions.prepPlaylistData(playlist), + + ) + +@app.route('/associate_track', methods=['POST']) +@functions.jellyfin_login_required +def associate_track(): + jellyfin_id = request.form.get('jellyfin_id') + provider_track_id = request.form.get('provider_track_id') + + if not jellyfin_id or not provider_track_id: + flash('Missing Jellyfin or Spotify ID') + + # Retrieve the track by Spotify ID + track = Track.query.filter_by(provider_track_id=provider_track_id).first() + + if not track: + flash('Track not found') + return '' + + # Associate the Jellyfin ID with the track + track.jellyfin_id = jellyfin_id + track.downloaded = True + + + try: + # Commit the changes to the database + db.session.commit() + flash("Track associated","success") + return '' + except Exception as e: + db.session.rollback() # Roll back the session in case of an error + flash(str(e)) + return '' + + +@app.route("/unlock_key",methods = ['POST']) +@functions.jellyfin_admin_required +def unlock_key(): + + key_name = request.form.get('inputLockKey') + if key_name: + tasks.release_lock(key_name) + flash(f'Lock {key_name} released', category='success') + return '' + + +@pl_bp.route('/test') +def test(): + #return '' + app.logger.info(f"performing full update on jellyfin track ids. (Update tracks and playlists if better quality will be found)") + downloaded_tracks : List[Track] = Track.query.all() + total_tracks = len(downloaded_tracks) + if not downloaded_tracks: + app.logger.info("No downloaded tracks without Jellyfin ID found.") + return {'status': 'No tracks to update'} + + app.logger.info(f"Found {total_tracks} tracks to update ") + processed_tracks = 0 + + for track in downloaded_tracks: + try: + best_match = tasks.find_best_match_from_jellyfin(track) + if best_match: + track.downloaded = True + if track.jellyfin_id != best_match['Id']: + track.jellyfin_id = best_match['Id'] + app.logger.info(f"Updated Jellyfin ID for track: {track.name} ({track.provider_track_id})") + if track.filesystem_path != best_match['Path']: + track.filesystem_path = best_match['Path'] + app.logger.info(f"Updated filesystem_path for track: {track.name} ({track.provider_track_id})") + + + + db.session.commit() + else: + app.logger.warning(f"No matching track found in Jellyfin for {track.name}.") + + spotify_track = None + + except Exception as e: + app.logger.error(f"Error searching Jellyfin for track {track.name}: {str(e)}") + + processed_tracks += 1 + progress = (processed_tracks / total_tracks) * 100 + #self.update_state(state=f'{processed_tracks}/{total_tracks}: {track.name}', meta={'current': processed_tracks, 'total': total_tracks, 'percent': progress}) + + app.logger.info("Finished updating Jellyfin IDs for all tracks.") + return {'status': 'All tracks updated', 'total': total_tracks, 'processed': processed_tracks} From cd498988aea91ab33b2569f7aaa490b4e9e3a04f Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:39:25 +0000 Subject: [PATCH 25/42] added dataclasses for combined information about track/playlist from provider and database --- app/classes.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/app/classes.py b/app/classes.py index 56a9bec..3c3ba67 100644 --- a/app/classes.py +++ b/app/classes.py @@ -1,7 +1,9 @@ +from dataclasses import dataclass +from datetime import datetime import subprocess import json from flask import current_app as app # Adjust this based on your app's structure -from typing import Optional +from typing import List, Optional class AudioProfile: @@ -77,3 +79,38 @@ class AudioProfile: """ return (f"AudioProfile(path='{self.path}', bitrate={self.bitrate} kbps, " f"sample_rate={self.sample_rate} Hz, channels={self.channels})") + + +@dataclass +class CombinedTrackData(): + # Combines a track from a provider with a track from the db + title: str + artist: List[str] + url: List[str] + duration: str + downloaded: bool + filesystem_path: Optional[str] + jellyfin_id: Optional[str] + provider_id: str + provider_track_id: str + duration_ms: int + download_status: Optional[str] + provider: str + +@dataclass +class CombinedPlaylistData(): + name: str + description: Optional[str] + image: str + url: str + id: str + jellyfin_id: Optional[str] + can_add: bool + can_remove: bool + last_updated: Optional[datetime] + last_changed: Optional[datetime] + tracks_available: int + track_count: int + tracks_linked: int + percent_available: float + status: str \ No newline at end of file From 00ba693fb9729aa75530729933ff122ff963985c Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:39:43 +0000 Subject: [PATCH 26/42] added jellyfin_link filter --- app/filters.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/filters.py b/app/filters.py index ac8c13f..f1ec19e 100644 --- a/app/filters.py +++ b/app/filters.py @@ -3,6 +3,7 @@ import re from markupsafe import Markup from app.classes import AudioProfile +from app import app filters = {} @@ -50,6 +51,17 @@ def audioprofile(text: str, path: str) -> Markup: f"Sample Rate: {audio_profile.sample_rate} Hz
" f"Channels: {audio_profile.channels}
" f"Quality Score: {audio_profile.compute_quality_score()}" - f"
" + ) return Markup(audio_profile_html) + + +@template_filter('jellyfin_link') +def jellyfin_link(jellyfin_id: str) -> Markup: + + jellyfin_server_url = app.config.get('JELLYFIN_SERVER_URL') + if not jellyfin_server_url: + return Markup(f"JELLYFIN_SERVER_URL not configured") + + link = f"{jellyfin_server_url}/web/#/details?id={jellyfin_id}" + return Markup(f'{jellyfin_id}') \ No newline at end of file From 2b3c400c10b75a65c57188a3510c2113da75bc93 Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:44:40 +0000 Subject: [PATCH 27/42] =?UTF-8?q?Major=20Overhaul:=20-=20No=20more=20dict?= =?UTF-8?q?=C2=B4s=20,=20goal=20is=20to=20have=20type=20safety=20and=20a?= =?UTF-8?q?=20generic=20approach=20to=20support=20multiple=20music=20(play?= =?UTF-8?q?list)=20providers=20-=20removed=20unneeded=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/functions.py | 316 ++++++++++----------------------------- app/models.py | 2 + app/providers/base.py | 50 ++++--- app/providers/spotify.py | 184 +++++++++++++++-------- app/tasks.py | 138 ++++++++--------- 5 files changed, 297 insertions(+), 393 deletions(-) diff --git a/app/functions.py b/app/functions.py index 3144ac0..9317648 100644 --- a/app/functions.py +++ b/app/functions.py @@ -1,12 +1,16 @@ import json -from typing import Optional -from flask import flash, redirect, session, url_for +from typing import List, Optional +from flask import flash, redirect, session, url_for,g import requests +from app.classes import CombinedPlaylistData, CombinedTrackData from app.models import JellyfinUser, Playlist,Track -from app import sp, cache, app, jellyfin ,jellyfin_admin_token, jellyfin_admin_id,device_id, cache +from app import sp, cache, app, jellyfin ,jellyfin_admin_token, jellyfin_admin_id,device_id, cache from functools import wraps from celery.result import AsyncResult -from app.tasks import download_missing_tracks,check_for_playlist_updates, update_all_playlists_track_status, update_jellyfin_id_for_downloaded_tracks +from app.providers import base +from app.providers.base import PlaylistTrack +from app.registry.music_provider_registry import MusicProviderRegistry +from . import tasks from jellyfin.objects import PlaylistMetadata from spotipy.exceptions import SpotifyException @@ -35,149 +39,63 @@ def manage_task(task_name): if result.state in ['PENDING', 'STARTED']: return result.state, result.info if result.info else {} if task_name == 'update_all_playlists_track_status': - result = update_all_playlists_track_status.delay() + result = tasks.update_all_playlists_track_status.delay() elif task_name == 'download_missing_tracks': - result = download_missing_tracks.delay() + result = tasks.download_missing_tracks.delay() elif task_name == 'check_for_playlist_updates': - result = check_for_playlist_updates.delay() + result = tasks.check_for_playlist_updates.delay() elif task_name == 'update_jellyfin_id_for_downloaded_tracks': - result = update_jellyfin_id_for_downloaded_tracks.delay() + result = tasks.update_jellyfin_id_for_downloaded_tracks.delay() TASK_STATUS[task_name] = result.id return result.state, result.info if result.info else {} -def prepPlaylistData(data): - playlists = [] +def prepPlaylistData(playlist: base.Playlist) -> Optional[CombinedPlaylistData]: jellyfin_user = JellyfinUser.query.filter_by(jellyfin_user_id=session['jellyfin_user_id']).first() if not jellyfin_user: app.logger.error(f"jellyfin_user not set: session user id: {session['jellyfin_user_id']}. Logout and Login again") return None - if not data.get('playlists'): - - data['playlists']= {} - data['playlists']['items'] = [data] - - for playlist_data in data['playlists']['items']: - # Fetch the playlist from the database if it exists - if playlist_data: - db_playlist = Playlist.query.filter_by(provider_playlist_id=playlist_data['id']).first() - if db_playlist: - # If the playlist is in the database, use the stored values - if playlist_data.get('tracks'): - if isinstance(playlist_data['tracks'],list): - track_count = len(playlist_data['tracks'] ) - else: - track_count = playlist_data['tracks']['total'] or 0 - else: - track_count = 0 - tracks_available = db_playlist.tracks_available or 0 - tracks_linked = len([track for track in db_playlist.tracks if track.jellyfin_id]) or 0 - percent_available = (tracks_available / track_count * 100) if track_count > 0 else 0 - - # Determine playlist status - if not playlist_data.get('status'): - if tracks_available == track_count and track_count > 0: - playlist_data['status'] = 'green' # Fully available - elif tracks_available > 0: - playlist_data['status'] = 'yellow' # Partially available - else: - playlist_data['status'] = 'red' # Not available - - else: - # If the playlist is not in the database, initialize with 0 - track_count = 0 - tracks_available = 0 - tracks_linked = 0 - percent_available = 0 - playlist_data['status'] = 'red' # Not requested yet + # Fetch the playlist from the database if it exists + db_playlist : Playlist = Playlist.query.filter_by(provider_playlist_id=playlist.id).first() if playlist else None - # Append playlist data to the list - playlists.append({ - 'name': playlist_data['name'], - 'description': playlist_data['description'], - 'image': playlist_data['images'][0]['url'] if playlist_data.get('images') else '/static/images/placeholder.png', - 'url': playlist_data['external_urls']['spotify'] if playlist_data.get('external_urls') else '', - 'id': playlist_data['id'] if playlist_data['id'] else '', - 'jellyfin_id': db_playlist.jellyfin_id if db_playlist else '', - 'can_add': (db_playlist not in jellyfin_user.playlists) if db_playlist else True, - 'can_remove' : (db_playlist in jellyfin_user.playlists) if db_playlist else False, - 'last_updated':db_playlist.last_updated if db_playlist else '', - 'last_changed':db_playlist.last_changed if db_playlist else '', - 'tracks_available': tracks_available, - 'track_count': track_count, - 'tracks_linked': tracks_linked, - 'percent_available': percent_available, - 'status': playlist_data['status'] # Red, yellow, or green based on availability - }) - - return playlists + # Initialize default values + track_count = db_playlist.track_count if db_playlist else 0 + tracks_available = db_playlist.tracks_available if db_playlist else 0 + tracks_linked = len([track for track in db_playlist.tracks if track.jellyfin_id]) if db_playlist else 0 + percent_available = (tracks_available / track_count * 100) if track_count > 0 else 0 -def get_cached_spotify_playlists(playlist_ids): - """ - Fetches multiple Spotify playlists by their IDs, utilizing individual caching. + # Determine playlist status + if tracks_available == track_count and track_count > 0: + status = 'green' # Fully available + elif tracks_available > 0: + status = 'yellow' # Partially available + else: + status = 'red' # Not available - :param playlist_ids: A list of Spotify playlist IDs. - :return: A dictionary containing the fetched playlists. - """ - spotify_data = {'playlists': {'items': []}} - - for playlist_id in playlist_ids: - playlist_data = None - not_found = False - try: - playlist_data = get_cached_spotify_playlist(playlist_id) - - except SpotifyException as e: - app.logger.error(f"Error Fetching Playlist {playlist_id}: {e}") - not_found = 'http status: 404' in str(e) - if not_found: - playlist_data = { - 'status':'red', - 'description': 'Playlist has most likely been removed. You can keep it, but won´t receive Updates.', - 'id': playlist_id, - 'name' : '' - - } + # Build and return the PlaylistResponse object + return CombinedPlaylistData( + name=playlist.name, + description=playlist.description, + image=playlist.images[0].url if playlist.images else '/static/images/placeholder.png', + url=playlist.external_urls[0].url if playlist.external_urls else '', + id=playlist.id, + jellyfin_id=db_playlist.jellyfin_id if db_playlist else '', + can_add=(db_playlist not in jellyfin_user.playlists) if db_playlist else True, + can_remove=(db_playlist in jellyfin_user.playlists) if db_playlist else False, + last_updated=db_playlist.last_updated if db_playlist else None, + last_changed=db_playlist.last_changed if db_playlist else None, + tracks_available=tracks_available, + track_count=track_count, + tracks_linked=tracks_linked, + percent_available=percent_available, + status=status + ) - if playlist_data: - spotify_data['playlists']['items'].append(playlist_data) - - return spotify_data -@cache.memoize(timeout=3600) -def get_cached_playlist(playlist_id): - """ - Fetches a Spotify playlist by its ID, utilizing caching to minimize API calls. - - :param playlist_id: The Spotify playlist ID. - :return: Playlist data as a dictionary, or None if an error occurs. - """ - # When the playlist_id starts with 37i9dQZF1, we need to use the new function - # as the standard Spotify API endpoints are deprecated for these playlists. - # Reference: https://github.com/kamilkosek/jellyplist/issues/25 - - if playlist_id.startswith("37i9dQZF1"): - app.logger.warning(f"Algorithmic or Spotify-owned editorial playlist, using custom Implementation to fetch details") - # Use the custom implementation for these playlists - try: - data = fetch_spotify_playlist(playlist_id) - return transform_playlist_response(data) - except Exception as e: - print(f"Error fetching playlist with custom method: {e}") - return None - - # Otherwise, use the standard Spotipy API - try: - playlist_data = sp.playlist(playlist_id) # Fetch data using Spotipy - return playlist_data - except Exception as e: - print(f"Error fetching playlist with Spotipy: {e}") - return None - -@cache.memoize(timeout=3600*24*10) -def get_cached_spotify_track(track_id): +@cache.memoize(timeout=3600*24*10) +def get_cached_provider_track(track_id : str,provider_id : str)-> base.Track: """ Fetches a Spotify track by its ID, utilizing caching to minimize API calls. @@ -185,86 +103,28 @@ def get_cached_spotify_track(track_id): :return: Track data as a dictionary, or None if an error occurs. """ try: - track_data = sp.track(track_id=track_id) # Fetch data from Spotify API + # get the provider from the registry + provider = MusicProviderRegistry.get_provider(provider_id) + track_data = provider.get_track(track_id) return track_data except Exception as e: - app.logger.error(f"Error fetching track {track_id} from Spotify: {str(e)}") + app.logger.error(f"Error fetching track {track_id} from {provider_id}: {str(e)}") return None -def prepAlbumData(data): - items = [] - for item in data['albums']['items']: - items.append({ - 'name': item['name'], - 'description': f"Released: {item['release_date']}", - 'image': item['images'][0]['url'] if item['images'] else 'default-image.jpg', - 'url': item['external_urls']['spotify'], - 'id' : item['id'], - 'can_add' : False - }) - return items - -def prepArtistData(data): - items = [] - for item in data['artists']['items']: - items.append({ - 'name': item['name'], - 'description': f"Popularity: {item['popularity']}", - 'image': item['images'][0]['url'] if item['images'] else 'default-image.jpg', - 'url': item['external_urls']['spotify'], - 'id' : item['id'], - 'can_add' : False - - }) - return items - - - -def getFeaturedPlaylists(country: str, offset: int): - try: - playlists_data = sp.featured_playlists(country=country, limit=16, offset=offset) - return prepPlaylistData(playlists_data), playlists_data['playlists']['total'], 'Featured Playlists' - except SpotifyException as e: - app.logger.error(f"Spotify API error in getFeaturedPlaylists: {e}") - return [], e, f'Error: Could not load featured playlists. Please try again later. This is most likely due to an Error in the Spotify API or an rate limit has been reached.' - -def getCategoryPlaylists(category: str, offset: int): - try: - playlists_data = sp.category_playlists(category_id=category, country=app.config['SPOTIFY_COUNTRY_CODE'], limit=16, offset=offset) - return prepPlaylistData(playlists_data), playlists_data['playlists']['total'], f"Category {playlists_data['message']}" - except SpotifyException as e: - app.logger.error(f"Spotify API error in getCategoryPlaylists: {e}") - return [], e, 'Error: Could not load category playlists. Please try again later. This is most likely due to an Error in the Spotify API or an rate limit has been reached.' - -def getCategories(country,offset): - categories_data = sp.categories(limit=16, offset= offset) - categories = [] - - for cat in categories_data['categories']['items']: - categories.append({ - 'name': cat['name'], - 'description': '', - 'image': cat['icons'][0]['url'] if cat['icons'] else 'default-image.jpg', - 'url': f"/playlists?cat={cat['id']}", - 'id' : cat['id'], - 'type':'category' - }) - return categories, categories_data['categories']['total'],'Browse Categories' - -def get_tracks_for_playlist(data): - results = data - tracks = [] +def get_tracks_for_playlist(data: List[PlaylistTrack], provider_id : str ) -> List[CombinedTrackData]: is_admin = session.get('is_admin', False) + tracks = [] - for idx, item in enumerate(results['tracks']['items']): - track_data = item['track'] + for idx, item in enumerate(data): + track_data = item.track if track_data: - duration_ms = track_data['duration_ms'] + duration_ms = track_data.duration_ms minutes = duration_ms // 60000 seconds = (duration_ms % 60000) // 1000 - track_db = Track.query.filter_by(provider_track_id=track_data['id']).first() + # Query track from the database + track_db = Track.query.filter_by(provider_track_id=track_data.id).first() if track_db: downloaded = track_db.downloaded @@ -277,40 +137,26 @@ def get_tracks_for_playlist(data): jellyfin_id = None download_status = None - tracks.append({ - 'title': track_data['name'], - 'artist': ', '.join([artist['name'] for artist in track_data['artists']]), - 'url': track_data['external_urls']['spotify'], - 'duration': f'{minutes}:{seconds:02d}', - 'preview_url': track_data['preview_url'], - 'downloaded': downloaded, - 'filesystem_path': filesystem_path, - 'jellyfin_id': jellyfin_id, - 'spotify_id': track_data['id'], - 'duration_ms': duration_ms, - 'download_status' : download_status - }) - + # Append a TrackResponse object + tracks.append( + CombinedTrackData( + title=track_data.name, + artist=[a.name for a in track_data.artists], + url=[url.url for url in track_data.external_urls], + duration=f'{minutes}:{seconds:02d}', + downloaded=downloaded, + filesystem_path=filesystem_path, + jellyfin_id=jellyfin_id, + provider_track_id=track_data.id, + provider_id = provider_id, + duration_ms=duration_ms, + download_status=download_status, + provider=provider_id + ) + ) + return tracks -def get_full_playlist_data(playlist_id): - playlist_data = get_cached_spotify_playlist(playlist_id) - all_tracks = [] - - offset = 0 - while True: - response = sp.playlist_items(playlist_id, offset=offset, limit=100) - items = response['items'] - all_tracks.extend(items) - - if len(items) < 100: - break - offset += 100 - - playlist_data['tracks'] = all_tracks - playlist_data['prepped_data'] = prepPlaylistData(playlist_data) - return playlist_data - def jellyfin_login_required(f): @wraps(f) def decorated_function(*args, **kwargs): @@ -332,13 +178,13 @@ def jellyfin_admin_required(f): -def update_playlist_metadata(playlist,spotify_playlist_data): +def update_playlist_metadata(playlist,provider_playlist_data : base.Playlist): metadata = PlaylistMetadata() metadata.Tags = [f'jellyplist:playlist:{playlist.id}',f'{playlist.tracks_available} of {playlist.track_count} Tracks available'] - metadata.Overview = spotify_playlist_data['description'] + metadata.Overview = provider_playlist_data.description jellyfin.update_playlist_metadata(session_token=_get_api_token(),playlist_id=playlist.jellyfin_id,updates= metadata , user_id= _get_admin_id()) - if spotify_playlist_data['images'] != None: - jellyfin.set_playlist_cover_image(session_token= _get_api_token(),playlist_id= playlist.jellyfin_id,spotify_image_url= spotify_playlist_data['images'][0]['url']) + if provider_playlist_data.images: + jellyfin.set_playlist_cover_image(session_token= _get_api_token(),playlist_id= playlist.jellyfin_id,provider_image_url= provider_playlist_data.images[0].url) @@ -347,7 +193,7 @@ def _get_token_from_sessioncookie() -> str: def _get_api_token() -> str: #return app.config['JELLYFIN_ACCESS_TOKEN'] return jellyfin_admin_token -def _get_logged_in_user(): +def _get_logged_in_user() -> JellyfinUser: return JellyfinUser.query.filter_by(jellyfin_user_id=session['jellyfin_user_id']).first() def _get_admin_id(): #return JellyfinUser.query.filter_by(is_admin=True).first().jellyfin_user_id diff --git a/app/models.py b/app/models.py index 76bac0f..da454e8 100644 --- a/app/models.py +++ b/app/models.py @@ -35,6 +35,7 @@ class Playlist(db.Model): snapshot_id = db.Column(db.String(120), nullable=True) # Many-to-Many relationship with JellyfinUser users = db.relationship('JellyfinUser', secondary=user_playlists, back_populates='playlists') + provider_id = db.Column(db.String(20)) def __repr__(self): return f'' @@ -56,6 +57,7 @@ class Track(db.Model): filesystem_path = db.Column(db.String(), nullable=True) jellyfin_id = db.Column(db.String(120), nullable=True) # Add Jellyfin track ID field download_status = db.Column(db.String(2048), nullable=True) + provider_id = db.Column(db.String(20)) # Many-to-Many relationship with Playlists playlists = db.relationship('Playlist', secondary=playlist_tracks, back_populates='tracks') diff --git a/app/providers/base.py b/app/providers/base.py index 6968a80..29850d6 100644 --- a/app/providers/base.py +++ b/app/providers/base.py @@ -82,6 +82,19 @@ class Playlist(ItemBase): images: Optional[List[Image]] owner: Optional[Owner] tracks: List[PlaylistTrack] = field(default_factory=list) +@dataclass +class BrowseCard: + title: str + uri: str + background_color: str + artwork: List[Image] + + +@dataclass +class BrowseSection: + title: str + items: List[BrowseCard] + uri: str # Abstract base class for music providers class MusicProviderClient(ABC): @@ -113,9 +126,19 @@ class MusicProviderClient(ABC): :return: A Playlist object. """ pass + @abstractmethod + def extract_playlist_id(self, uri: str) -> str: + """ + Extracts the playlist ID from a playlist URI. + :param uri: The playlist URI. + :return: The playlist ID. + """ + pass + + @abstractmethod - def search_tracks(self, query: str, limit: int = 50) -> List[Track]: + def search_playlist(self, query: str, limit: int = 50) -> List[Playlist]: """ Searches for tracks based on a query string. :param query: The search query. @@ -133,29 +156,18 @@ class MusicProviderClient(ABC): """ pass @abstractmethod - def get_featured_playlists(self, limit: int = 50) -> List[Playlist]: + def browse(self, **kwargs) -> List[BrowseSection]: """ - Fetches a list of featured playlists. - :param limit: Maximum number of featured playlists to return. - :return: A list of Playlist objects. + Generic browse method for the music provider. + :param kwargs: Variable keyword arguments to support different browse parameters + :return: A dictionary containing browse results """ pass - @abstractmethod - def get_playlists_by_category(self, category_id: str, limit: int = 50) -> List[Playlist]: + def browse_page(self, uri: str) -> List[Playlist]: """ - Fetches playlists belonging to a specific category. - :param category_id: The ID of the category. - :param limit: Maximum number of playlists to return. + Fetches a specific page of browse results. + :param uri: The uri to query. :return: A list of Playlist objects. """ - pass - - @abstractmethod - def get_categories(self, limit: int = 50) -> List[Category]: - """ - Fetches a list of available categories. - :param limit: Maximum number of categories to return. - :return: A list of categories, where each category is a dictionary with 'id' and 'name'. - """ pass \ No newline at end of file diff --git a/app/providers/spotify.py b/app/providers/spotify.py index 83f1f06..07c8da9 100644 --- a/app/providers/spotify.py +++ b/app/providers/spotify.py @@ -1,6 +1,6 @@ from dataclasses import dataclass import os -from app.providers.base import AccountAttributes, Album, Artist, Image, MusicProviderClient, Owner,Playlist, PlaylistTrack, Profile,Track,ExternalUrl,Category +from app.providers.base import AccountAttributes, Album, Artist, BrowseCard, BrowseSection, Image, MusicProviderClient, Owner, Playlist, PlaylistTrack, Profile, Track, ExternalUrl, Category import requests import json @@ -13,21 +13,6 @@ import logging l = logging.getLogger(__name__) -@dataclass -class BrowseCard: - title: str - uri: str - background_color: str - artwork: List[Image] - - -@dataclass -class BrowseSection: - title: str - items: List[BrowseCard] - uri: str - - class SpotifyClient(MusicProviderClient): """ Spotify implementation of the MusicProviderClient. @@ -201,12 +186,15 @@ class SpotifyClient(MusicProviderClient): :param album_data: Dictionary representing an album. :return: An Album instance. """ + artists = [] + if album_data.get("artists"): + artists = [self._parse_artist(artist) for artist in album_data.get("artists").get('items', [])] return Album( id=album_data["uri"].split(":")[-1], name=album_data["name"], uri=album_data["uri"], external_urls=self._parse_external_urls(album_data["uri"], "album"), - artists=[self._parse_artist(artist) for artist in album_data["artists"]["items"]], + artists=artists, images=self._parse_images(album_data["coverArt"]["sources"]) ) @@ -218,15 +206,33 @@ class SpotifyClient(MusicProviderClient): :param track_data: Dictionary representing a track. :return: A Track instance. """ + duration_ms = 0 + aritsts = [] + if track_data.get("duration"): + duration_ms = int(track_data.get("duration", 0).get("totalMilliseconds", 0)) + elif track_data.get("trackDuration"): + duration_ms = track_data["trackDuration"]["totalMilliseconds"] + + if track_data.get("firstArtist"): + for artist in track_data.get("firstArtist").get('items', []): + aritsts.append(self._parse_artist(artist)) + elif track_data.get("artists"): + for artist in track_data.get("artists").get('items', []): + aritsts.append(self._parse_artist(artist)) + + if track_data.get("albumOfTrack"): + album = self._parse_album(track_data["albumOfTrack"]) + + return Track( id=track_data["uri"].split(":")[-1], name=track_data["name"], uri=track_data["uri"], external_urls=self._parse_external_urls(track_data["uri"], "track"), - duration_ms=track_data["trackDuration"]["totalMilliseconds"], + duration_ms=duration_ms, explicit=track_data.get("explicit", False), album=self._parse_album(track_data["albumOfTrack"]), - artists=[self._parse_artist(artist) for artist in track_data["artists"]["items"]] + artists=aritsts ) def _parse_owner(self, owner_data: Dict) -> Optional[Owner]: """ @@ -283,10 +289,14 @@ class SpotifyClient(MusicProviderClient): owner_data = playlist_data.get("ownerV2", {}).get("data", {}) owner = self._parse_owner(owner_data) - tracks = [ - self._parse_track(item["itemV2"]["data"]) - for item in playlist_data.get("content", {}).get("items", []) - ] + + valid_tracks = [] + for item in playlist_data.get("content", {}).get("items", []): + data = item.get("itemV2", {}).get("data", {}) + uri = data.get("uri", "") + if uri.startswith("spotify:track"): + valid_tracks.append(self._parse_track(data)) + tracks = valid_tracks return Playlist( id=playlist_data.get("uri", "").split(":")[-1], @@ -369,16 +379,61 @@ class SpotifyClient(MusicProviderClient): playlist_data["content"]["items"] = all_items return self._parse_playlist(playlist_data) - - def search_tracks(self, query: str, limit: int = 10) -> List[Track]: + + def extract_playlist_id(self, uri: str) -> str: """ - Searches for tracks on Spotify. + Extract the playlist ID from a Spotify URI. + """ + # check whether the uri is a full url with https or just a uri + if uri.startswith("https://open.spotify.com/"): + #if it starts with https, we need to extract the playlist id from the url + return uri.split('/')[-1] + elif uri.startswith("spotify:playlist:"): + return uri.split(':')[-1] + else : + raise ValueError("Invalid Spotify URI.") + + def search_playlist(self, query: str, limit: int = 50) -> List[Playlist]: + """ + Searches for playlists on Spotify. :param query: Search query. :param limit: Maximum number of results. - :return: A list of Track objects. + :return: A list of Playlist objects. """ - print(f"search_tracks: Placeholder for search with query '{query}' and limit {limit}.") - return [] + query_parameters = { + "operationName": "searchDesktop", + "variables": json.dumps({ + "searchTerm": query, + "offset": 0, + "limit": limit, + "numberOfTopResults": 5, + "includeAudiobooks": False, + "includeArtistHasConcertsField": False, + "includePreReleases": False, + "includeLocalConcertsField": False + }), + "extensions": json.dumps({ + "persistedQuery": { + "version": 1, + "sha256Hash": "f1f1c151cd392433ef4d2683a10deb9adeefd660f29692d8539ce450d2dfdb96" + } + }) + } + encoded_query = urlencode(query_parameters) + url = f"pathfinder/v1/query?{encoded_query}" + + try: + response = self._make_request(url) + search_data = response.get("data", {}).get("searchV2", {}) + playlists_data = search_data.get("playlists", {}).get("items", []) + + playlists = [self._parse_playlist(item["data"]) for item in playlists_data] + return playlists + + except Exception as e: + print(f"An error occurred while searching for playlists: {e}") + return [] + def get_track(self, track_id: str) -> Track: """ @@ -386,37 +441,30 @@ class SpotifyClient(MusicProviderClient): :param track_id: The ID of the track. :return: A Track object. """ - print(f"get_track: Placeholder for track with ID {track_id}.") - return Track(id=track_id, name="", uri="", duration_ms=0, explicit=False, album=Album(), artists=[], external_urls= ExternalUrl()) + query_parameters = { + "operationName": "getTrack", + "variables": json.dumps({ + "uri": f"spotify:track:{track_id}" + }), + "extensions": json.dumps({ + "persistedQuery": { + "version": 1, + "sha256Hash": "5c5ec8c973a0ac2d5b38d7064056c45103c5a062ee12b62ce683ab397b5fbe7d" + } + }) + } + encoded_query = urlencode(query_parameters) + url = f"pathfinder/v1/query?{encoded_query}" - def get_featured_playlists(self, limit: int = 10) -> List[Playlist]: - """ - Fetches featured playlists. - :param limit: Maximum number of results. - :return: A list of Playlist objects. - """ - print(f"get_featured_playlists: Placeholder for featured playlists with limit {limit}.") - return [] + try: + response = self._make_request(url) + track_data = response.get("data", {}).get("trackUnion", {}) + return self._parse_track(track_data) - def get_playlists_by_category(self, category_id: str, limit: int = 10) -> List[Playlist]: - """ - Fetches playlists for a specific category. - :param category_id: The ID of the category. - :param limit: Maximum number of results. - :return: A list of Playlist objects. - """ - print(f"get_playlists_by_category: Placeholder for playlists in category {category_id}.") - return [] + except Exception as e: + print(f"An error occurred while fetching the track: {e}") + return None - def get_categories(self, limit: int = 10) -> List[Category]: - """ - Fetches categories from Spotify. - :param limit: Maximum number of results. - :return: A list of Category objects. - """ - print(f"get_categories: Placeholder for categories with limit {limit}.") - return [] - # non generic method implementations: def get_profile(self) -> Optional[Profile]: @@ -506,14 +554,17 @@ class SpotifyClient(MusicProviderClient): except Exception as e: print(f"An error occurred while fetching account attributes: {e}") return None - def browse_all(self, page_limit: int = 50, section_limit: int = 99) -> List[BrowseSection]: + def browse(self, **kwargs) -> List[BrowseSection]: """ Fetch all browse sections with cards. - :param page_limit: Maximum number of pages to fetch. - :param section_limit: Maximum number of sections per page. + :param kwargs: Keyword arguments. Supported: + - page_limit: Maximum number of pages to fetch (default: 50) + - section_limit: Maximum number of sections per page (default: 99) :return: A list of BrowseSection objects. """ + page_limit = kwargs.get('page_limit', 50) + section_limit = kwargs.get('section_limit', 99) query_parameters = { "operationName": "browseAll", "variables": json.dumps({ @@ -541,22 +592,23 @@ class SpotifyClient(MusicProviderClient): print(f"An error occurred while fetching browse sections: {e}") return [] - def browse_page(self, card: BrowseCard) -> List[Playlist]: + def browse_page(self, uri: str) -> List[Playlist]: """ - Fetch the content of a browse page using the URI from a BrowseCard. + Fetch the content of a browse page using the URI. - :param card: A BrowseCard instance with a URI starting with 'spotify:page'. + :param uri: Should start with 'spotify:page'. :return: A list of Playlist objects from the browse page. """ - if not card.uri.startswith("spotify:page"): - raise ValueError("The BrowseCard URI must start with 'spotify:page'.") + + if not uri or not uri.startswith("spotify:page"): + raise ValueError("The 'uri' parameter must be provided and start with 'spotify:page'.") query_parameters = { "operationName": "browsePage", "variables": json.dumps({ "pagePagination": {"offset": 0, "limit": 10}, "sectionPagination": {"offset": 0, "limit": 10}, - "uri": card.uri + "uri": uri }), "extensions": json.dumps({ "persistedQuery": { diff --git a/app/tasks.py b/app/tasks.py index a8084d6..f9c11c7 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -1,6 +1,7 @@ from datetime import datetime,timezone import logging import subprocess +from typing import List from sqlalchemy import insert from app import celery, app, db, functions, sp, jellyfin, jellyfin_admin_token, jellyfin_admin_id @@ -10,8 +11,8 @@ from app.models import JellyfinUser,Playlist,Track, user_playlists, playlist_tra import os import redis from celery import current_task,signals -import asyncio -import requests + +from app.registry.music_provider_registry import MusicProviderRegistry redis_client = redis.StrictRedis(host='redis', port=6379, db=0) def acquire_lock(lock_name, expiration=60): @@ -96,7 +97,8 @@ def download_missing_tracks(self): client_secret = app.config['SPOTIFY_CLIENT_SECRET'] search_before_download = app.config['SEARCH_JELLYFIN_BEFORE_DOWNLOAD'] - undownloaded_tracks = Track.query.filter_by(downloaded=False).all() + # Downloading using SpotDL only works for Spotify tracks + undownloaded_tracks : List[Track] = Track.query.filter_by(downloaded=False,provider_id = "Spotify").all() total_tracks = len(undownloaded_tracks) if not undownloaded_tracks: app.logger.info("No undownloaded tracks found.") @@ -113,7 +115,7 @@ def download_missing_tracks(self): # region search before download if search_before_download: app.logger.info(f"Searching for track in Jellyfin: {track.name}") - spotify_track = functions.get_cached_spotify_track(track.provider_track_id) + spotify_track = functions.get_cached_provider_track(track.provider_track_id, provider_id="Spotify") # at first try to find the track without fingerprinting it best_match = find_best_match_from_jellyfin(track) if best_match: @@ -123,39 +125,39 @@ def download_missing_tracks(self): app.logger.info(f"Updated Jellyfin ID for track: {track.name} ({track.provider_track_id})") if track.filesystem_path != best_match['Path']: track.filesystem_path = best_match['Path'] - - db.session.commit() - processed_tracks+=1 + db.session.commit() + processed_tracks+=1 continue # region search with fingerprinting - if spotify_track: - preview_url = spotify_track.get('preview_url') - if not preview_url: - app.logger.error(f"Preview URL not found for track {track.name}.") - # Decide whether to skip or proceed to download - # For now, we'll proceed to download - else: - # Get the list of Spotify artist names - spotify_artists = [artist['name'] for artist in spotify_track['artists']] + # as long as there is no endpoint found providing a preview url, we can't use this feature + # if spotify_track: + # preview_url = spotify_track.get('preview_url') + # if not preview_url: + # app.logger.error(f"Preview URL not found for track {track.name}.") + # # Decide whether to skip or proceed to download + # # For now, we'll proceed to download + # else: + # # Get the list of Spotify artist names + # spotify_artists = [artist['name'] for artist in spotify_track['artists']] - # Perform the search in Jellyfin - match_found, jellyfin_file_path = jellyfin.search_track_in_jellyfin( - session_token=jellyfin_admin_token, - preview_url=preview_url, - song_name=track.name, - artist_names=spotify_artists - ) - if match_found: - app.logger.info(f"Match found in Jellyfin for track {track.name}. Skipping download.") - track.downloaded = True - track.filesystem_path = jellyfin_file_path - db.session.commit() - continue - else: - app.logger.info(f"No match found in Jellyfin for track {track.name}. Proceeding to download.") - else: - app.logger.warning(f"spotify_track not set, see previous log messages") + # # Perform the search in Jellyfin + # match_found, jellyfin_file_path = jellyfin.search_track_in_jellyfin( + # session_token=jellyfin_admin_token, + # preview_url=preview_url, + # song_name=track.name, + # artist_names=spotify_artists + # ) + # if match_found: + # app.logger.info(f"Match found in Jellyfin for track {track.name}. Skipping download.") + # track.downloaded = True + # track.filesystem_path = jellyfin_file_path + # db.session.commit() + # continue + # else: + # app.logger.info(f"No match found in Jellyfin for track {track.name}. Proceeding to download.") + # else: + # app.logger.warning(f"spotify_track not set, see previous log messages") #endregion #endregion @@ -240,7 +242,7 @@ def check_for_playlist_updates(self): try: app.logger.info('Starting playlist update check...') with app.app_context(): - playlists = Playlist.query.all() + playlists: List[Playlist] = Playlist.query.all() total_playlists = len(playlists) if not playlists: app.logger.info("No playlists found.") @@ -251,40 +253,28 @@ def check_for_playlist_updates(self): for playlist in playlists: playlist.last_updated = datetime.now( timezone.utc) - sp_playlist = sp.playlist(playlist.provider_playlist_id) + # get the correct MusicProvider from the registry + provider = MusicProviderRegistry.get_provider(playlist.provider_id) + provider_playlist = provider.get_playlist(playlist.provider_playlist_id) + provider_tracks = provider_playlist.tracks full_update = True - app.logger.info(f'Checking updates for playlist: {playlist.name}, s_snapshot = {sp_playlist['snapshot_id']}') + app.logger.info(f'Checking updates for playlist: {playlist.name}') db.session.commit() - if sp_playlist['snapshot_id'] == playlist.snapshot_id: - app.logger.info(f'playlist: {playlist.name} , no changes detected, snapshot_id {sp_playlist['snapshot_id']}') - full_update = False + try: #region Check for updates - # Fetch all playlist data from Spotify if full_update: - spotify_tracks = {} - offset = 0 - playlist.snapshot_id = sp_playlist['snapshot_id'] - while True: - playlist_data = sp.playlist_items(playlist.provider_playlist_id, offset=offset, limit=100) - items = playlist_data['items'] - spotify_tracks.update({offset + idx: track['track'] for idx, track in enumerate(items) if track['track']}) - - if len(items) < 100: # No more tracks to fetch - break - offset += 100 # Move to the next batch - existing_tracks = {track.provider_track_id: track for track in playlist.tracks} # Determine tracks to add and remove tracks_to_add = [] - for idx, track_info in spotify_tracks.items(): + for idx, track_info in enumerate(provider_tracks): if track_info: - track_id = track_info['id'] + track_id = track_info.track.id if track_id not in existing_tracks: - track = Track.query.filter_by(provider_track_id=track_id).first() + track = Track.query.filter_by(provider_track_id=track_id,provider_id = playlist.provider_id).first() if not track: - track = Track(name=track_info['name'], provider_track_id=track_id, provider_uri=track_info['uri'], downloaded=False) + track = Track(name=track_info.track.name, provider_track_id=track_id, provider_uri=track_info.track.uri, downloaded=False,provider_id = playlist.provider_id) db.session.add(track) db.session.commit() app.logger.info(f'Added new track: {track.name}') @@ -293,7 +283,7 @@ def check_for_playlist_updates(self): tracks_to_remove = [ existing_tracks[track_id] for track_id in existing_tracks - if track_id not in {track['id'] for track in spotify_tracks.values() if track} + if track_id not in {track.track.id for track in provider_tracks if track} ] if tracks_to_add or tracks_to_remove: @@ -321,7 +311,7 @@ def check_for_playlist_updates(self): #endregion #region Update Playlist Items and Metadata - functions.update_playlist_metadata(playlist, sp_playlist) + functions.update_playlist_metadata(playlist, provider_playlist) ordered_tracks = db.session.execute( db.select(Track, playlist_tracks.c.track_order) .join(playlist_tracks, playlist_tracks.c.track_id == Track.id) @@ -414,45 +404,47 @@ def update_jellyfin_id_for_downloaded_tracks(self): def find_best_match_from_jellyfin(track: Track): app.logger.debug(f"Trying to find best match from Jellyfin server for track: {track.name}") search_results = jellyfin.search_music_tracks(jellyfin_admin_token, functions.get_longest_substring(track.name)) - spotify_track = None + provider_track = None try: best_match = None best_quality_score = -1 # Initialize with the lowest possible score - - for result in search_results: - app.logger.debug(f"Processing search result: {result['Id']}, Path = {result['Path']}") quality_score = compute_quality_score(result, app.config['FIND_BEST_MATCH_USE_FFPROBE']) try: - spotify_track = functions.get_cached_spotify_track(track.provider_track_id) - spotify_track_name = spotify_track['name'].lower() - spotify_artists = [artist['name'].lower() for artist in spotify_track['artists']] + provider_track = functions.get_cached_provider_track(track.provider_track_id, provider_id=track.provider_id) + provider_track_name = provider_track.name.lower() + provider_artists = [artist.name.lower() for artist in provider_track.artists] except Exception as e: app.logger.error(f"\tError fetching track details from Spotify for {track.name}: {str(e)}") continue jellyfin_track_name = result.get('Name', '').lower() - jellyfin_artists = [artist.lower() for artist in result.get('Artists', [])] + if len(result.get('Artists', [])) == 1: + jellyfin_artists = [a.lower() for a in result.get('Artists', [])[0].split('/')] + else: + jellyfin_artists = [artist.lower() for artist in result.get('Artists', [])] + jellyfin_album_artists = [artist['Name'].lower() for artist in result.get('AlbumArtists', [])] - if spotify_track and jellyfin_track_name and jellyfin_artists and spotify_artists: + if provider_track and jellyfin_track_name and jellyfin_artists and provider_artists: app.logger.debug("\tTrack details to compare: ") app.logger.debug(f"\t\tJellyfin-Trackname : {jellyfin_track_name}") - app.logger.debug(f"\t\t Spotify-Trackname : {spotify_track_name}") + app.logger.debug(f"\t\t Spotify-Trackname : {provider_track_name}") app.logger.debug(f"\t\t Jellyfin-Artists : {jellyfin_artists}") - app.logger.debug(f"\t\t Spotify-Artists : {spotify_artists}") + app.logger.debug(f"\t\t Spotify-Artists : {provider_artists}") + app.logger.debug(f"\t\t Jellyfin-Alb.Art.: {jellyfin_album_artists}") if len(search_results) == 1: app.logger.debug(f"\tOnly 1 search_result: {result['Id']} ({app.config['JELLYFIN_SERVER_URL']}/web/#/details?id={result['Id']})") - if (spotify_track_name.lower() == jellyfin_track_name and - set(artist.lower() for artist in spotify_artists) == set(jellyfin_artists)): + if (provider_track_name.lower() == jellyfin_track_name and + (set(artist.lower() for artist in provider_artists) == set(jellyfin_artists) or set(jellyfin_album_artists) == set(artist.lower() for artist in provider_artists))): app.logger.debug(f"\tQuality score for track {result['Name']}: {quality_score} [{result['Path']}]") best_match = result break - if (spotify_track_name.lower() == jellyfin_track_name and - set(artist.lower() for artist in spotify_artists) == set(jellyfin_artists)): + if (provider_track_name.lower() == jellyfin_track_name and + (set(artist.lower() for artist in provider_artists) == set(jellyfin_artists) or set(jellyfin_album_artists) == set(artist.lower() for artist in provider_artists))): app.logger.debug(f"\tQuality score for track {result['Name']}: {quality_score} [{result['Path']}]") if quality_score > best_quality_score: From 883294d74ee84a72336bb7ecfc17460f6616efc0 Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:45:13 +0000 Subject: [PATCH 28/42] added get_item to jellyfin-client --- jellyfin/client.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/jellyfin/client.py b/jellyfin/client.py index e2e5c06..ae4ab0a 100644 --- a/jellyfin/client.py +++ b/jellyfin/client.py @@ -315,6 +315,18 @@ class JellyfinClient: return {"status": "success", "message": "Playlist removed successfully"} else: raise Exception(f"Failed to remove playlist: {response.content}") + + def get_item(self, session_token: str, item_id: str): + url = f'{self.base_url}/Items/{item_id}' + self.logger.debug(f"Url={url}") + + response = requests.get(url, headers=self._get_headers(session_token=session_token), timeout = self.timeout) + self.logger.debug(f"Response = {response.status_code}") + + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to get item: {response.content}") def remove_user_from_playlist(self, session_token: str, playlist_id: str, user_id: str): """ @@ -341,7 +353,7 @@ class JellyfinClient: raise Exception(f"Failed to remove user from playlist: {response.content}") - def set_playlist_cover_image(self, session_token: str, playlist_id: str, spotify_image_url: str): + def set_playlist_cover_image(self, session_token: str, playlist_id: str, provider_image_url: str): """ Set the cover image of a playlist in Jellyfin using an image URL from Spotify. @@ -351,7 +363,7 @@ class JellyfinClient: :return: Success message or raises an exception on failure. """ # Step 1: Download the image from the Spotify URL - response = requests.get(spotify_image_url, timeout = self.timeout) + response = requests.get(provider_image_url, timeout = self.timeout) if response.status_code != 200: raise Exception(f"Failed to download image from Spotify: {response.content}") @@ -444,7 +456,6 @@ class JellyfinClient: return response.json() - def search_track_in_jellyfin(self, session_token: str, preview_url: str, song_name: str, artist_names: list): """ Search for a track in Jellyfin by comparing the preview audio to tracks in the library. @@ -546,8 +557,6 @@ class JellyfinClient: print(f"Error in search_track_in_jellyfin: {str(e)}") return False, None - - # Helper methods used in search_track_in_jellyfin def download_preview_to_tempfile(self, preview_url): try: From 87791cf21d0792a8e3bca619d21915ce23fd8115 Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 12:48:27 +0000 Subject: [PATCH 29/42] - Support multiple music providers - feat: Doubleclick on track in Table view to get technical information about it --- static/images/placeholder.png | Bin 0 -> 8056 bytes static/js/preview.js | 10 +- templates/base.html | 60 +++++--- templates/browse.html | 61 ++++++++ templates/browse_page.html | 12 ++ templates/monitored_playlists.html | 16 ++ templates/partials/_add_remove_button.html | 2 +- templates/partials/_jf_search_results.html | 2 +- templates/partials/_playlist_info.html | 10 +- templates/partials/_track_table.html | 170 ++++++++++++--------- templates/partials/playlist_item.html | 49 ++++++ templates/partials/track_details.html | 23 +++ 12 files changed, 306 insertions(+), 109 deletions(-) create mode 100644 static/images/placeholder.png create mode 100644 templates/browse.html create mode 100644 templates/browse_page.html create mode 100644 templates/monitored_playlists.html create mode 100644 templates/partials/playlist_item.html create mode 100644 templates/partials/track_details.html diff --git a/static/images/placeholder.png b/static/images/placeholder.png new file mode 100644 index 0000000000000000000000000000000000000000..12516164e3cd1ef7f2c3b86734100d178279e440 GIT binary patch literal 8056 zcmcgxS2WyFxBt%=j55j)y^R(H*sZ#*?X29 z<`y7M3OWh^P@O<>X@h^Ov)QRX(g6T}ZUDf91Hivq6J`Sd(82()Weot*nE*iR@vd24 z_SONm)lyRiZXy$g+HW;NFAW_P!VN;`9di064LBzNuwXQl6%2jn{=A<};ph)sdLVVt zHbyu$?cwd{@96Y=Ph+pr_amLThkO3&xe=N z+AkfTY?3CIsvDM%!YSY3r}l@pQC%k}`5+*!H}BeuOmYUYx+HEy4nFBUt2}%nMY8B(RN&vh67-XxFxSJCA$DeS{x_RGul0$wPII3s-+HCO0YxB$0>uWQfPx_Z z1relh>~;N-!qYaK$DjbE@$cM?l1(!gG3U=?kcXnw`gb8f!Pbe2%h@g?b zP!i<7RUScy%u^0w$KSk|(7-^ECoXAd0`-m4Nl|u;N(eYYw2DTT8ML!23|4D^~fWB?sUGS!~i_(Xq>8y;EvC3et<_o8rJn_XW1Q(^TDWV%F zB%SUH6ZTv&rRL2+`gsgMt@(i%bl{MPvvqYp_t>y-A!*Bs^&Qc1Z$=VD%m^1s7j4y- zLV#6E7qB6qcSx-i-I2%G8hu8Pf`-eC-l%qW8##GYMecw{uHs3;iJD zpFBuC%h^?~#9;Ua+eNn7>}VSfT`ZY$moxQSLL zjEk;H8D0W!vV zO{34RLnsO8SK$F4lO2}~1XE4zU>i=E`UKmVZ@7q=ykfI_T?|o%PN2&~xkav*Lk;Lg z`28;TV`Gv#qMl8=JBliTrtF}gdfbDm7FL0txmJtYBU(Nt7H=S6x7CZZB<3|%X3_1; zMi_8_+-Ua^J|~&5qX=n73s;^*yPQwljs?>@iu;Q_2JE2DIsC`Sp5R0EW>(u+q-vII zg@BZ{HmN95D;>%a>znzeK^B^m<*Uhw%7A~xzy1mZw-4+iUro!q4lzde#MobPs{dH! zp6gz4x+lyj#RbpHCUB(xd(nYRa!=OKT~8rN)bzf zNS#q(5c}Ne6OqskQNg9VO7ARp_ZKKL-%x4magV}dgc!oTiWSVD9(B0j1VJUj67%L> z0t*4pBs><+Hg5W&#hGrlu*dcpGOKk-rZ8qg4rw}hIf$tQ6Mw}A-Gw!IEmSI)rZIj& zTlRS?>-eG-4VdW@iRnSVpn!GVCOLnNb`GiDN=SijI5^SsH!(9yMYCn%m{7qCpeB7i z*t76{atyz2#x?@xPrtJ6W$f#MSWP+#r+~)L9Y%R8Y(HWHGh~ z>}%~}<5kElQqQ7Tn2`ccW0rR*8$5DDpG}t*E9yf%n1rl<|LSouhM0^rji77(jh|wP znD!}aj{#4Pw&1r+9?Ntke_lVEJ~=U7rbhKrv6ZWwB(pepGx>qZeeHWQpQ}FFP}^jx(Dw-=xHI<&((nA2+FK$o$;f zPvE=T6kTzA&?mg+|9QNa&>H^-c*6s%d>myi+G8kg(byqagDxQKn!8_-{B0KbFlCG~ zlo<5SwCZL1zrHP*;|XRI*sGLy#wJT(w9I#@hC|1dI_BYqLu^nFmULp|$BhMysai@& zTcj^Avh-&OZIec_I9whFX;B?iMRyl)LY!%7TxD&tU~C1rD>FMdPmi!4@oFFF3#rJ6 zkZa{`!tg zkwj`4RH}mQ4xQhZbhQS(D9)FtV%^G-kgZNj>u+~KEUxfxCk$K&)S$Oly1=%0dac~A zw{{N`oMMQCp?dX{d=ny+-aMe+*(<_Sj|phx?!4B)u-MsK9gZIsP@0$X#T8I=R1tpB zPQTlubj*#KqPszGOO?KiaEEn6_48tgyl3;3qo_@I5-Trs{+`%$4iuV}ta9Kyy87_x zCU8q|uvnJ_zrEQrX93tI^qQ(L2Hwr&KTUFnvb7K2){Bz6lYl(%gA?;uML~$e6y0BO9o7(^lF z14YL~w*~v)Ba(lfstLhb4?NWMy#;=iClGP>c~*QN+eG%&?WGZ2?OCJYI8O!rqh-ko0T$b$-J!1yDjt0As^X3TnFT99#iAdd$k>^bV=^I1t^;x&o$!}@S~r|Cw8>E=>_7Dtj_ zj(F%=mpV6DEbb@9;?hkhiO9 zs@0H-OhAkaH#V3gPYx`5Amv{YuXnFj7Y%QccIAbX67@0lfcvD3!9_eQxaj%PlBNgC zuh;ix#GWJ&PT|Yi+{;@O_mz__PtRw7uoO)v-SPNgx~%d{MDZPu&50cTWVHN0%XV&e z0b24Je2-4ZhP&{|_g9y+%vl}1WdL79-ao%jU++6=GN?A_TP+)dAv|P>ZiA$CMA#zc z&lJL&@YZYevQ$|2NofO*L8r(rbIJS%UhqgaV4dEbd-{ou0y9xb!S-vGbNm-@+s^214Wa0mJUUoY0^3) zHro*1572T~>T(G58Sgn2H|n9zuzN7>2K9&7@aqr6;||NhhEn%aD>n{Red{oy*E6E! zA*7oZP*A)&fs`F4{RzpcB&?exMx3&lC&M-Ri$rjuO|mkxYwFIDF_}m8Mm#&pFxQ=U zmqoR&@hYm&E1)UiAe~pwHq3n}4fQ?`04GwWvh4;C`)GcSOXr~SCFXe%FSOF*qQGOL z-&3HSHJ7Z7U{{Ws4g@S8uQ_TZMd%ds))rVyF^{DIAP!U%(X`0$8nN}aYPdDcUNl>z zAoct={Aq$8ag>GANt06f+ZhCY6a9by*+~_|u?}3S3y!q-%V~aa|FwlwK$K>5`C^Uz zvSkN`!n)Imqx^R|?J+WjM9qAp%-gV074s1>8_EJ9BL%I<_s+(WHM`L%EaMv?&ZMvt ziVZKKK!>YwFZ8P8*-&g#_?%|02mWqzN~{6P6>l@PoCV0=EJOxZ@!T_VCbff&A@-|nS>H&u7O~>S?+>FG#A{W#=WzhBymv}L7hk?kMm8Z~k zh6qEo*qht@#?)N;V*RmP@87N$gLYoz5{A>+Wz#3n;o$u4>j;UvsPUPahKG?U5r91G zYoxp&(4vUC9c=ITVY@{hl;l!RXuF7(^sQ8@tTu1jhCCWRBX1y~V*_)(rO1QtA_aj8 zBA9iItk0Wd`|PLwQHSjFt21t{KgX&FtSbSOj0@EwI z#-W<&m<HCeT`Oda5*RNqlx4M5!w-ceb-==OU_zZ7&j8z8$3qU_N?c2OvGkiA{4 z`)aNaOlsuylf3}psbPo|SrXjUF2L*Rl5szoBhh+Ds)|~YF9YsF)i$O;dKYAe6rVyE z?1kJ(3O$IDgo2Pi^VMJbeb-s6E7ZfQ-{XH~7pqa6`REzPYQMf9AuTPKX@shjt3ycA zS5?v{-OM^dY_k}B9p9x+s;@WR4KdJ8S*||VEJ~;UO@=@@3+k1-gaz-raPDnbC#Eov z6qy~<50i_(Vs$y%;o;(dUfxOj+2G^mM_BR>*ncnmxau&g_r*-t^KS0x0m{P(OOn%Z@wCp{L%gC@xS$KxA)_Q|2Q>=1R;Ky3+0y1_fVc z$!+kQ6s{c-+5N2BXB+3Y76)5P=#uu0Ce7)r57_(xWN~8Fq5EG00hJ zfZB`PpyJF)Bm6l@s;k%=CqFY4t`}?&DW>%Eamt^9ipgdp<0ie^`;skVu>^bNlbDNM z?AG!SW!_L@0wP&rNFb$Xt0|6gD8+bwtWh^is=Z=&UPH**nh0POUx@!LykVQn>lkN= z(%DIJ$nj%+zropQ@Jg4e|Mr){l5jqIaP0;k60zvjt}}JaQDJ>ns1w%sc_u{2nkycz z&Y5t2oreFLJZnWXmZAc=&7HY?=TJao$wT9*!*Te@z&G%1`%u**XT^nUa(&}3OCD|@ zqU6&2*kiG=E|S4p^TXcTrmoN!>;(a_`3XQaI5eX5GZ4=Dc^yJFw%smk429arZ`*Fd z4|^uu-;MXUYCt^<)jhYNdo+%zp4@K@K|-~o@`!@&s7YUvF4Cjc^_@=b-uN4U$~sEO zO2%K(gqx};)A?7V@hT{qASwIe{kklTR*^<687ZN+(>~c>aGs}26st^d6m@?(-<98B zJRPzC9X;+|I}l#OCCv*f!s7%u=Q&aDklgC^U-+{!%uDZ*|BF3aVY#gfK3x9#O#C38 zKqVSItc>}d#3O*f6h6h}w4nW4&>G3;f&jHQ{*Egk8D<*L4gl`>Rt(wT(ioV@S~tO) z*N&w?#H!?niD0Co0o>!|Am#}Hg=G7ffFNdZP3DbLA8OFDpY4KYY?TnVy2$t=dM5?7 z<`9WPS=;w;B%G68erju35%@~4WXkg2TnL4HeUlL_xG|jR|Mi1>CRtYv8l3=4Gw~{6O!r>h)g=eZxDSoKD(D@dY5MZdqZO6L zQIIk9Qsyi9(w5U*Fh6j7aL@3xOW4Yw$Gm5KO+=zO6Y@9Cy-ag~!1mpd)CB>}*f66P zom#%)5{*_lhZqq`#Wn6C*|TYc^&Vxh-7Wa zn>IclJ^au$N~Le1Y$YmxVR#j&yQsecxJ!qP^2Ygv)veRV%$azsylbE^{~E%O>w}Io z(QYY`qwWyV0*1BKHIX?Il;Id52uAvzKF65u;`JWPP-BcJyS{2dIeeF#+P*@Y*PQ|$jGC` zi=l50k;?tW;-N#uR%NJfT9(=_Ss6I20_cNa&3MZse3hrCJ5(?Je5V&JT`4Bhqq^rN zgttaIUr+#4qT*oKQugGA*SQ9uve^}mU4(VR*zhsL^u20w~+3$m>|ZJejM=^a>*8SA|}IkQCgmj{mTuL0spmuuHw z=qNt>7(E(1)Ex7cV~+FXTj!Bci}Ov|G(_fY{>uoJfr0)obd5diXzz8jW-Kp}9#YY2 znfzQS$fOnX7ymnLn#r{sEl9{kX@F=R=Gwd6kcRmK`mthTInQ2Y3!T`y8r{%_Jj3;s zoM(!8jDFLZzYgkt=xfA!^=9j}P>u_(C5K%zNsJ#=mk;)}V0@dn9(XeAw|xI+3F*Qb zZQEl71d1yB3?3YhUs;yQB45FBdD_&=^kv86ZRFNO@YYiZ&B^-1nZp@JiIMxe29AVM zd&piB!;k{`pCU{XA`cu0n}%DI=~lUdh)rL;wHQL8s^js2m{-esK8nD@BHtV)B&!t4*_}9Y;;JVW-g3$NFtryEciss*kzDH|soT?Y z@Mq$wAo}SX#`@%YMT(2b{vZ0P7TM(KKhNVnKQjMfZXev6b+;qoJa}u;^|IN&en1lB zn^Yw=GF)CCRCUOOUL^`HeiFB@rkWwU5-e5uZsjz@0quSA&wODexWeSNg&U=wQ{gP5 zSp2ir{2Y&~TQ0?c_7PX{Q*!00S+me>*eZJ92 zX_NEIGWXA{(X1-6)fMFX%Py^qQvx-Qa}NdW+TJxchVHj992tRH3X*H8ob?}6P2snW zkcVuy7Sc2Sb`UD%;}|`B+cs~&i$6XrdP&{1mNnoRUxgPSa1dC}cV0PS{w#jMjNe%jZUy-FZiT*0W+Jvi9gceN>??_~M`BKHxm?b9#&-Gi@WN~TZ$ zydicbKN?tR%d0v18|U!TBrVR$@|hW;OPJT)Wa-f{y zq?)`$AhLYGKg$+{cC9DL_1*~VISmBl?kK#$+Ie{4U*!Ls)VFc#iJ5eww#jC5>;kCRK zaaxeu@sW^EKOPCSv@NqRBBn70=}7f1ZnvUdDMADtMGWo(69i z7cG7e^|Xa|uzV$Y;vu^wkZ7YLNW^HLEw*3A2mVb4iZ~Mw>l%D)0AD>xpSLr)+5QE? zZ78vn-@fy$e1*0tzp_`pD`L2@6?^jArd;z~jYg+!pI~3>$Ur(9X`$X7+`q?4{2Z{B z>*#VL8S6=-c`A|kh&epjK@OE_t@yv%eA!Q4Qdp;uW8nAY0?{KUY^?WJCz?3)WS_tC z>9(8NkEQ4c`2B~=i1DK)rs5T=mKlkOC5Z8K@`|(HcKYCg-^hZR=4`ZeWIa}tmMKMB zRMw(IPSmTmWG8K_aL(fgILuV1zp*90Gi~%f)LQ7--hEH-yF{3^$r7_ygx){H%@5*o z_4B1_lE2Dq)R console.error('Error fetching Jellyfin stream URL:', error)); } -function handleJellyfinClick(event, jellyfinId, trackTitle, spotifyId) { +function handleJellyfinClick(event, jellyfinId, trackTitle, providerTrackId) { if (event.ctrlKey) { // CTRL key is pressed, open the search modal - openSearchModal(trackTitle, spotifyId); + openSearchModal(trackTitle, providerTrackId); } else { // CTRL key is not pressed, play the track playJellyfinTrack(event.target, jellyfinId); diff --git a/templates/base.html b/templates/base.html index 6203a42..93bb6b7 100644 --- a/templates/base.html +++ b/templates/base.html @@ -34,12 +34,22 @@
  • + Home +
    • + {% for provider in registered_providers %} +
  • + + Browse {{provider}} + +
    • + {% endfor %} +
  • Monitored @@ -108,29 +128,23 @@ -

    {{ title }}

    - -
    -
    -
    - - -
    -
    -
    - - {{ session.get('jellyfin_user_name') }} -
    -
    + +
    + + +
    + + {{ session.get('jellyfin_user_name') }}
- +

{{ title }}

+

{{ subtitle }}

{% block content %}{% endblock %} diff --git a/templates/browse.html b/templates/browse.html new file mode 100644 index 0000000..e76cd6f --- /dev/null +++ b/templates/browse.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} + +{% block content %} +{% for section in browse_data %} +
+

{{ section.title }}

+
+ {% for card in section.items %} +
+
+ + {{ card.title }} +
+
{{ card.title }}
+
+ +
+
+ {% endfor %} +
+
+{% endfor %} +{% endblock %} + \ No newline at end of file diff --git a/templates/browse_page.html b/templates/browse_page.html new file mode 100644 index 0000000..779aee3 --- /dev/null +++ b/templates/browse_page.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} + +{% block content %} + + +
+ {% for item in data %} + {% include 'partials/playlist_item.html' %} + {% endfor %} +
+{% endblock %} \ No newline at end of file diff --git a/templates/monitored_playlists.html b/templates/monitored_playlists.html new file mode 100644 index 0000000..f919d7f --- /dev/null +++ b/templates/monitored_playlists.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} + +{% block content %} +
+ {% for provider_id, playlists in provider_playlists_data.items() %} +
+

{{ provider_id }}

+
+ {% for item in playlists %} + {% include 'partials/playlist_item.html' %} + {% endfor %} +
+
+ {% endfor %} +
+{% endblock %} \ No newline at end of file diff --git a/templates/partials/_add_remove_button.html b/templates/partials/_add_remove_button.html index cd91833..70d6faf 100644 --- a/templates/partials/_add_remove_button.html +++ b/templates/partials/_add_remove_button.html @@ -1,5 +1,5 @@ {% if item.can_add %} - - diff --git a/templates/partials/_playlist_info.html b/templates/partials/_playlist_info.html index a38f482..656ae86 100644 --- a/templates/partials/_playlist_info.html +++ b/templates/partials/_playlist_info.html @@ -1,13 +1,13 @@
- +
-

{{ playlist_name }}

-

{{ playlist_description }}

-

{{ track_count }} songs, {{ total_duration }}

-

Last Updated: {{ last_updated}} | Last Change: {{ last_changed}}

+

{{ item.name }}

+

{{ item.description }}

+

{{ item.track_count }} songs, {{ total_duration }}

+

Last Updated: {{ item.last_updated}} | Last Change: {{ item.last_changed}}

{% include 'partials/_add_remove_button.html' %}
diff --git a/templates/partials/_track_table.html b/templates/partials/_track_table.html index a450053..1f66e74 100644 --- a/templates/partials/_track_table.html +++ b/templates/partials/_track_table.html @@ -1,75 +1,98 @@ - - - - - - - - - - - - - - {% for track in tracks %} - - - - - - - - - + + + + + + + + + + + + + {% for track in tracks %} + + + + + + + + + - - {% endfor %} - -
#TitleArtistDurationSpotifyPreviewStatusJellyfin
{{ loop.index }}{{ track.title }}{{ track.artist }}{{ track.duration }} - - - - - {% if track.preview_url %} - - {% else %} - - - - {% endif %} - - {% if not track.downloaded %} - - {% else %} - - {% endif %} - - {% set title = track.title | replace("'","") %} +
#TitleArtistDuration{{provider_id}}PreviewStatusJellyfin
{{ loop.index }}{{ track.title }}{{ track.artist }}{{ track.duration }} + + + + + {% if track.preview_url %} + + {% else %} + + + + {% endif %} + + {% if not track.downloaded %} + + {% else %} + + {% endif %} + + {% set title = track.title | replace("'","") %} - {% if track.jellyfin_id %} - + {% elif track.downloaded %} + + - {% elif track.downloaded %} - - - - {% else %} - - - - {% endif %} -
+ + {% else %} + + + + {% endif %} + + + {% endfor %} + + + + + - +
- - - + \ No newline at end of file diff --git a/templates/partials/playlist_item.html b/templates/partials/playlist_item.html new file mode 100644 index 0000000..475499c --- /dev/null +++ b/templates/partials/playlist_item.html @@ -0,0 +1,49 @@ +
+
+ + + {% if item.status %} + + {% if item.track_count > 0 %} + {{ item.tracks_available }} / {{ item.tracks_linked}} / {{ item.track_count}} + {% else %} + not Available + {% endif %} + + {% endif %} + + +
+ {{ item.name }} +
+ + +
+
+
{{ item.name }}
+

{{ item.description }}

+
+
+ {% if item.type == 'category'%} + + + + {%else%} + + + + + {%endif%} + {% include 'partials/_add_remove_button.html' %} +
+
+ +
+
\ No newline at end of file diff --git a/templates/partials/track_details.html b/templates/partials/track_details.html new file mode 100644 index 0000000..e222b7c --- /dev/null +++ b/templates/partials/track_details.html @@ -0,0 +1,23 @@ +
+ + +
+
+

Title: {{ track.title }}

+

Artist: {{ track.artist }}

+

Duration: {{ track.duration }}

+

Duration (ms): {{ track.duration_ms }}

+

Provider: {{ track.provider_id }}

+

Provider Track URL: {{track.provider_track_url}}

+ +

Status: {{ 'Downloaded' if track.downloaded else 'Not Downloaded' }}

+

Jellyfin ID: {{ track.jellyfin_id | jellyfin_link }}

+

Provider Track ID: {{ track.provider_track_id }}

+

Download Status: {{ track.download_status }}

+

Filesystem Path: {{ track.filesystem_path }}

+

Jellyfin Filesystem Path: {{ track.jellyfin_filesystem_path if track.jellyfin_filesystem_path else 'N/A' }}

+

{{ track.jellyfin_filesystem_path | audioprofile(track.jellyfin_filesystem_path) if track.jellyfin_filesystem_path else 'N/A' }}

+
+
+ +
\ No newline at end of file From b861a1a8f4b42e42a8bc9d380ccf0abdb1be393b Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 23:11:05 +0000 Subject: [PATCH 30/42] feat: added lidarr support --- app/__init__.py | 24 +-- app/functions.py | 51 ++++- app/models.py | 1 + app/routes/routes.py | 70 +++---- app/tasks.py | 93 +++++++++- config.py | 4 + lidarr/__init__.py | 2 + lidarr/classes.py | 174 ++++++++++++++++++ lidarr/client.py | 143 ++++++++++++++ .../d13088ebddc5_add_lidarr_processed_flag.py | 32 ++++ templates/admin.html | 3 + templates/admin/lidarr.html | 32 ++++ 12 files changed, 570 insertions(+), 59 deletions(-) create mode 100644 lidarr/__init__.py create mode 100644 lidarr/classes.py create mode 100644 lidarr/client.py create mode 100644 migrations/versions/d13088ebddc5_add_lidarr_processed_flag.py create mode 100644 templates/admin/lidarr.html diff --git a/app/__init__.py b/app/__init__.py index 12e751e..03e226f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -8,6 +8,7 @@ from flask import Flask, has_request_context from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate from psycopg2 import OperationalError +import redis import spotipy from spotipy.oauth2 import SpotifyClientCredentials from celery import Celery @@ -77,9 +78,13 @@ def make_celery(app): 'update_jellyfin_id_for_downloaded_tracks-schedule': { 'task': 'app.tasks.update_jellyfin_id_for_downloaded_tracks', 'schedule': crontab(minute='*/10'), - } } + if app.config['LIDARR_API_KEY']: + celery.conf.beat_schedule['request-lidarr-schedule'] = { + 'task': 'app.tasks.request_lidarr', + 'schedule': crontab(minute='*/15') + } celery.conf.timezone = 'UTC' return celery @@ -89,17 +94,6 @@ device_id = f'JellyPlist_{'_'.join(sys.argv)}' # Initialize Flask app app = Flask(__name__, template_folder="../templates", static_folder='../static') -# log_file = 'app.log' -# handler = RotatingFileHandler(log_file, maxBytes=100000, backupCount=3) -# handler.setLevel(logging.DEBUG) -# handler.setFormatter(log_formatter) -# stream_handler = logging.StreamHandler(sys.stdout) -# stream_handler.setLevel(logging.DEBUG) -# stream_handler.setFormatter(log_formatter) - - -# # app.logger.addHandler(handler) -# app.logger.addHandler(stream_handler) app.config.from_object(Config) @@ -114,6 +108,7 @@ logging.basicConfig(format=FORMAT) Config.validate_env_vars() cache = Cache(app) +redis_client = redis.StrictRedis(host=app.config['CACHE_REDIS_HOST'], port=app.config['CACHE_REDIS_PORT'], db=0, decode_responses=True) # Spotify, Jellyfin, and Spotdl setup @@ -181,3 +176,8 @@ spotify_client = SpotifyClient('/jellyplist/open.spotify.com_cookies.txt') spotify_client.authenticate() from .registry import MusicProviderRegistry MusicProviderRegistry.register_provider(spotify_client) + +if app.config['LIDARR_API_KEY'] and app.config['LIDARR_URL']: + app.logger.info(f'Creating Lidarr Client with URL: {app.config["LIDARR_URL"]}') + from lidarr.client import LidarrClient + lidarr_client = LidarrClient(app.config['LIDARR_URL'], app.config['LIDARR_API_KEY']) diff --git a/app/functions.py b/app/functions.py index 9317648..c188087 100644 --- a/app/functions.py +++ b/app/functions.py @@ -4,12 +4,13 @@ from flask import flash, redirect, session, url_for,g import requests from app.classes import CombinedPlaylistData, CombinedTrackData from app.models import JellyfinUser, Playlist,Track -from app import sp, cache, app, jellyfin ,jellyfin_admin_token, jellyfin_admin_id,device_id, cache +from app import sp, cache, app, jellyfin ,jellyfin_admin_token, jellyfin_admin_id,device_id, cache, redis_client from functools import wraps from celery.result import AsyncResult from app.providers import base from app.providers.base import PlaylistTrack from app.registry.music_provider_registry import MusicProviderRegistry +from lidarr.classes import Album, Artist from . import tasks from jellyfin.objects import PlaylistMetadata from spotipy.exceptions import SpotifyException @@ -20,15 +21,20 @@ TASK_STATUS = { 'update_all_playlists_track_status': None, 'download_missing_tracks': None, 'check_for_playlist_updates': None, - 'update_jellyfin_id_for_downloaded_tracks' : None + 'update_jellyfin_id_for_downloaded_tracks' : None, + + } +if app.config['LIDARR_API_KEY']: + TASK_STATUS['request_lidarr'] = None + LOCK_KEYS = [ 'update_all_playlists_track_status_lock', 'download_missing_tracks_lock', 'check_for_playlist_updates_lock', 'update_jellyfin_id_for_downloaded_tracks_lock' , - 'full_update_jellyfin_ids' - + 'full_update_jellyfin_ids', + 'request_lidarr_lock' ] def manage_task(task_name): @@ -46,6 +52,8 @@ def manage_task(task_name): result = tasks.check_for_playlist_updates.delay() elif task_name == 'update_jellyfin_id_for_downloaded_tracks': result = tasks.update_jellyfin_id_for_downloaded_tracks.delay() + elif task_name == 'request_lidarr': + result = tasks.request_lidarr.delay() TASK_STATUS[task_name] = result.id return result.state, result.info if result.info else {} @@ -93,6 +101,41 @@ def prepPlaylistData(playlist: base.Playlist) -> Optional[CombinedPlaylistData]: status=status ) +def lidarr_quality_profile_id(profile_id=None): + if app.config['LIDARR_API_KEY'] and app.config['LIDARR_URL']: + from app import lidarr_client + if profile_id: + redis_client.set('lidarr_quality_profile_id', profile_id) + else: + value = redis_client.get('lidarr_quality_profile_id') + if not value: + value = lidarr_client.get_quality_profiles()[0] + lidarr_quality_profile_id(value.id) + return value + return value + +def lidarr_root_folder_path(folder_path=None): + if app.config['LIDARR_API_KEY'] and app.config['LIDARR_URL']: + from app import lidarr_client + if folder_path: + redis_client.set('lidarr_root_folder_path', folder_path) + else: + value = redis_client.get('lidarr_root_folder_path') + if not value: + value = lidarr_client.get_root_folders()[0] + lidarr_root_folder_path(value.path) + return value.path + return value + +# a function which takes a lidarr.class.Artist object as a parameter, and applies the lidarr_quality_profile_id to the artist if its 0 +def apply_default_profile_and_root_folder(object : Artist ) -> Artist: + if object.qualityProfileId == 0: + object.qualityProfileId = int(lidarr_quality_profile_id()) + if object.rootFolderPath == '' or object.rootFolderPath == None: + object.rootFolderPath = str(lidarr_root_folder_path()) + if object.metadataProfileId == 0: + object.metadataProfileId = 1 + return object @cache.memoize(timeout=3600*24*10) def get_cached_provider_track(track_id : str,provider_id : str)-> base.Track: diff --git a/app/models.py b/app/models.py index da454e8..42f4dc4 100644 --- a/app/models.py +++ b/app/models.py @@ -61,5 +61,6 @@ class Track(db.Model): # Many-to-Many relationship with Playlists playlists = db.relationship('Playlist', secondary=playlist_tracks, back_populates='tracks') + lidarr_processed = db.Column(db.Boolean(), default=False) def __repr__(self): return f'' diff --git a/app/routes/routes.py b/app/routes/routes.py index 0ee8e3b..ef9da3d 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -1,8 +1,9 @@ +from dbm import error import json import os import re from flask import Flask, Response, jsonify, render_template, request, redirect, url_for, session, flash, Blueprint, g -from app import app, db, functions, sp, jellyfin, celery, jellyfin_admin_token, jellyfin_admin_id,device_id, cache, read_dev_build_file, tasks +from app import app, db, functions, jellyfin, read_dev_build_file, tasks from app.classes import AudioProfile, CombinedPlaylistData from app.models import JellyfinUser,Playlist,Track from celery.result import AsyncResult @@ -12,6 +13,8 @@ from app.providers import base from app.providers.base import MusicProviderClient from app.providers.spotify import SpotifyClient from app.registry.music_provider_registry import MusicProviderRegistry +from lidarr.classes import Album, Artist +from lidarr.client import LidarrClient from ..version import __version__ from spotipy.exceptions import SpotifyException from collections import defaultdict @@ -37,6 +40,29 @@ def render_messages(response: Response) -> Response: return response +@app.route('/admin/lidarr') +@functions.jellyfin_admin_required +def admin_lidarr(): + if app.config['LIDARR_API_KEY'] and app.config['LIDARR_URL']: + from app import lidarr_client + q_profiles = lidarr_client.get_quality_profiles() + root_folders = lidarr_client.get_root_folders() + return render_template('admin/lidarr.html',quality_profiles = q_profiles, root_folders = root_folders, current_quality_profile = functions.lidarr_quality_profile_id(), current_root_folder = functions.lidarr_root_folder_path()) + return render_template('admin/lidarr.html', error = 'Lidarr not configured') + +@app.route('/admin/lidarr/save', methods=['POST']) +@functions.jellyfin_admin_required +def save_lidarr_config(): + quality_profile_id = request.form.get('qualityProfile') + root_folder_id = request.form.get('rootFolder') + + if not quality_profile_id or not root_folder_id: + flash('Both Quality Profile and Root Folder must be selected', 'danger') + return redirect(url_for('admin_lidarr')) + functions.lidarr_quality_profile_id(quality_profile_id) + functions.lidarr_root_folder_path(root_folder_id) + flash('Configuration saved successfully', 'success') + return redirect(url_for('admin_lidarr')) @app.route('/admin/tasks') @functions.jellyfin_admin_required @@ -370,43 +396,5 @@ def unlock_key(): @pl_bp.route('/test') def test(): - #return '' - app.logger.info(f"performing full update on jellyfin track ids. (Update tracks and playlists if better quality will be found)") - downloaded_tracks : List[Track] = Track.query.all() - total_tracks = len(downloaded_tracks) - if not downloaded_tracks: - app.logger.info("No downloaded tracks without Jellyfin ID found.") - return {'status': 'No tracks to update'} - - app.logger.info(f"Found {total_tracks} tracks to update ") - processed_tracks = 0 - - for track in downloaded_tracks: - try: - best_match = tasks.find_best_match_from_jellyfin(track) - if best_match: - track.downloaded = True - if track.jellyfin_id != best_match['Id']: - track.jellyfin_id = best_match['Id'] - app.logger.info(f"Updated Jellyfin ID for track: {track.name} ({track.provider_track_id})") - if track.filesystem_path != best_match['Path']: - track.filesystem_path = best_match['Path'] - app.logger.info(f"Updated filesystem_path for track: {track.name} ({track.provider_track_id})") - - - - db.session.commit() - else: - app.logger.warning(f"No matching track found in Jellyfin for {track.name}.") - - spotify_track = None - - except Exception as e: - app.logger.error(f"Error searching Jellyfin for track {track.name}: {str(e)}") - - processed_tracks += 1 - progress = (processed_tracks / total_tracks) * 100 - #self.update_state(state=f'{processed_tracks}/{total_tracks}: {track.name}', meta={'current': processed_tracks, 'total': total_tracks, 'percent': progress}) - - app.logger.info("Finished updating Jellyfin IDs for all tracks.") - return {'status': 'All tracks updated', 'total': total_tracks, 'processed': processed_tracks} + return '' + \ No newline at end of file diff --git a/app/tasks.py b/app/tasks.py index f9c11c7..d9059d2 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -4,7 +4,7 @@ import subprocess from typing import List from sqlalchemy import insert -from app import celery, app, db, functions, sp, jellyfin, jellyfin_admin_token, jellyfin_admin_id +from app import celery, app, db, functions, sp, jellyfin, jellyfin_admin_token, jellyfin_admin_id, redis_client from app.classes import AudioProfile from app.models import JellyfinUser,Playlist,Track, user_playlists, playlist_tracks @@ -12,9 +12,10 @@ import os import redis from celery import current_task,signals +from app.providers import base from app.registry.music_provider_registry import MusicProviderRegistry +from lidarr.classes import Artist -redis_client = redis.StrictRedis(host='redis', port=6379, db=0) def acquire_lock(lock_name, expiration=60): return redis_client.set(lock_name, "locked", ex=expiration, nx=True) @@ -401,6 +402,94 @@ def update_jellyfin_id_for_downloaded_tracks(self): app.logger.info("Skipping task. Another instance is already running.") return {'status': 'Task skipped, another instance is running'} +@celery.task(bind=True) +def request_lidarr(self): + lock_key = "request_lidarr_lock" + + if acquire_lock(lock_key, expiration=600): + with app.app_context(): + if app.config['LIDARR_API_KEY'] and app.config['LIDARR_URL']: + from app import lidarr_client + try: + app.logger.info('Submitting request to Lidarr...') + # get all tracks from db + tracks = Track.query.filter_by(lidarr_processed=False).all() + total_items = len(tracks) + processed_items = 0 + for track in tracks: + tfp = functions.get_cached_provider_track(track.provider_track_id, provider_id=track.provider_id) + if tfp: + if app.config['LIDARR_MONITOR_ARTISTS']: + app.logger.debug("Monitoring artists instead of albums") + # get all artists from all tracks_from_provider and unique them + artists : dict[str,base.Artist] = {} + + for artist in tfp.artists: + artists[artist.name] = artist + app.logger.debug(f"Found {len(artists)} artists to monitor") + #pylint: disable=consider-using-dict-items + for artist in artists: + artist_from_lidarr = None + search_result = lidarr_client.search(artists[artist].name) + for url in artists[artist].external_urls: + artist_from_lidarr : Artist = lidarr_client.get_object_by_external_url(search_result, url.url) + if artist_from_lidarr: + app.logger.debug(f"Found artist {artist_from_lidarr.artistName} by external url {url.url}") + functions.apply_default_profile_and_root_folder(artist_from_lidarr) + try: + lidarr_client.monitor_artist(artist_from_lidarr) + track.lidarr_processed = True + db.session.commit() + except Exception as e: + app.logger.error(f"Error monitoring artist {artist_from_lidarr.artistName}: {str(e)}") + + if not artist_from_lidarr: + # if the artist isnt found by the external url, search by name + artist_from_lidarr = lidarr_client.get_artists_by_name(search_result, artists[artist].name) + for artist2 in artist_from_lidarr: + functions.apply_default_profile_and_root_folder(artist2) + try: + lidarr_client.monitor_artist(artist2) + track.lidarr_processed = True + db.session.commit() + except Exception as e: + app.logger.error(f"Error monitoring artist {artist2.artistName}: {str(e)}") + + processed_items += 1 + self.update_state(state=f'{processed_items}/{total_items}: {artist}', meta={'current': processed_items, 'total': total_items, 'percent': (processed_items / total_items) * 100}) + + else: + if tfp.album: + album_from_lidarr = None + search_result = lidarr_client.search(tfp.album.name) + # if the album isnt found by the external url, search by name + album_from_lidarr = lidarr_client.get_albums_by_name(search_result, tfp.album.name) + for album2 in album_from_lidarr: + functions.apply_default_profile_and_root_folder(album2.artist) + try: + lidarr_client.monitor_album(album2) + track.lidarr_processed = True + db.session.commit() + except Exception as e: + app.logger.error(f"Error monitoring album {album2.title}: {str(e)}") + processed_items += 1 + self.update_state(state=f'{processed_items}/{total_items}: {tfp.album.name}', meta={'current': processed_items, 'total': total_items, 'percent': (processed_items / total_items) * 100}) + + + app.logger.info(f'Requests sent to Lidarr. Total items: {total_items}') + return {'status': 'Request sent to Lidarr'} + finally: + release_lock(lock_key) + + else: + app.logger.info('Lidarr API key or URL not set. Skipping request.') + release_lock(lock_key) + + + else: + app.logger.info("Skipping task. Another instance is already running.") + return {'status': 'Task skipped, another instance is running'} + def find_best_match_from_jellyfin(track: Track): app.logger.debug(f"Trying to find best match from Jellyfin server for track: {track.name}") search_results = jellyfin.search_music_tracks(jellyfin_admin_token, functions.get_longest_substring(track.name)) diff --git a/config.py b/config.py index ca151e2..3eb6027 100644 --- a/config.py +++ b/config.py @@ -27,6 +27,10 @@ class Config: SEARCH_JELLYFIN_BEFORE_DOWNLOAD = os.getenv('SEARCH_JELLYFIN_BEFORE_DOWNLOAD',"true").lower() == 'true' FIND_BEST_MATCH_USE_FFPROBE = os.getenv('FIND_BEST_MATCH_USE_FFPROBE','false').lower() == 'true' SPOTIFY_COUNTRY_CODE = os.getenv('SPOTIFY_COUNTRY_CODE','DE') + LIDARR_API_KEY = os.getenv('LIDARR_API_KEY','') + LIDARR_URL = os.getenv('LIDARR_URL','') + LIDARR_MONITOR_ARTISTS = os.getenv('LIDARR_MONITOR_ARTISTS','false').lower() == 'true' + # SpotDL specific configuration SPOTDL_CONFIG = { 'cookie_file': '/jellyplist/cookies.txt', diff --git a/lidarr/__init__.py b/lidarr/__init__.py new file mode 100644 index 0000000..2a984da --- /dev/null +++ b/lidarr/__init__.py @@ -0,0 +1,2 @@ +from .client import LidarrClient +__all__ = ["LidarrClient"] \ No newline at end of file diff --git a/lidarr/classes.py b/lidarr/classes.py new file mode 100644 index 0000000..940be03 --- /dev/null +++ b/lidarr/classes.py @@ -0,0 +1,174 @@ +from dataclasses import dataclass, field +from typing import List, Optional + +@dataclass +class Image: + url: str + coverType: str + extension: str + remoteUrl: str + +@dataclass +class Link: + url: str + name: str + +@dataclass +class Ratings: + votes: int + value: float + +@dataclass +class AddOptions: + monitor: str + albumsToMonitor: List[str] + monitored: bool + searchForMissingAlbums: bool + +@dataclass +class Statistics: + albumCount: int + trackFileCount: int + trackCount: int + totalTrackCount: int + sizeOnDisk: int + percentOfTracks: float + +@dataclass +class Member: + name: str + instrument: str + images: List[Image] + +@dataclass +class Artist: + mbId: Optional[str] = None + tadbId: Optional[int] = None + discogsId: Optional[int] = None + allMusicId: Optional[str] = None + overview: str = "" + artistType: str = "" + disambiguation: str = "" + links: List[Link] = field(default_factory=list) + nextAlbum: str = "" + lastAlbum: str = "" + images: List[Image] = field(default_factory=list) + members: List[Member] = field(default_factory=list) + remotePoster: str = "" + path: str = "" + qualityProfileId: int = 0 + metadataProfileId: int = 0 + monitored: bool = False + monitorNewItems: str = "" + rootFolderPath: Optional[str] = None + folder: str = "" + genres: List[str] = field(default_factory=list) + cleanName: str = "" + sortName: str = "" + tags: List[int] = field(default_factory=list) + added: str = "" + addOptions: Optional[AddOptions] = None + ratings: Optional[Ratings] = None + statistics: Optional[Statistics] = None + status : str = "" + ended : bool = False + artistName : str = "" + foreignArtistId : str = "" + id : int = 0 + + +@dataclass +class Media: + mediumNumber: int + mediumName: str + mediumFormat: str + +@dataclass +class Release: + id: int + albumId: int + foreignReleaseId: str + title: str + status: str + duration: int + trackCount: int + media: List[Media] + mediumCount: int + disambiguation: str + country: List[str] + label: List[str] + format: str + monitored: bool + +@dataclass +class Album: + id: int = 0 + title: str = "" + disambiguation: str = "" + overview: str = "" + artistId: int = 0 + foreignAlbumId: str = "" + monitored: bool = False + anyReleaseOk: bool = False + profileId: int = 0 + duration: int = 0 + albumType: str = "" + secondaryTypes: List[str] = field(default_factory=list) + mediumCount: int = 0 + ratings: Ratings = None + releaseDate: str = "" + releases: List[Release] = field(default_factory=list) + genres: List[str] = field(default_factory=list) + media: List[Media] = field(default_factory=list) + artist: Artist = field(default_factory=Artist) + images: List[Image] = field(default_factory=list) + links: List[Link] = field(default_factory=list) + lastSearchTime: str = "" + statistics: Statistics = None + addOptions: Optional[dict] = field(default_factory=dict) + remoteCover: str = "" + +@dataclass +class RootFolder: + id: int = 0 + name: str = "" + path: str = "" + defaultMetadataProfileId: int = 0 + defaultQualityProfileId: int = 0 + defaultMonitorOption: str = "" + defaultNewItemMonitorOption: str = "" + defaultTags: List[int] = field(default_factory=list) + accessible: bool = False + freeSpace: int = 0 + totalSpace: int = 0 + +@dataclass +class Quality: + id: int = 0 + name: str = "" + +@dataclass +class Item: + id: int = 0 + name: str = "" + quality: Quality = field(default_factory=Quality) + items: List[str] = field(default_factory=list) + allowed: bool = False + +@dataclass +class FormatItem: + id: int = 0 + format: int = 0 + name: str = "" + score: int = 0 + +@dataclass +class QualityProfile: + id: int = 0 + name: str = "" + upgradeAllowed: bool = False + cutoff: int = 0 + items: List[Item] = field(default_factory=list) + minFormatScore: int = 0 + cutoffFormatScore: int = 0 + formatItems: List[FormatItem] = field(default_factory=list) \ No newline at end of file diff --git a/lidarr/client.py b/lidarr/client.py new file mode 100644 index 0000000..48feee2 --- /dev/null +++ b/lidarr/client.py @@ -0,0 +1,143 @@ +import json +import re +from flask import jsonify +import requests +from typing import List, Optional +from .classes import Album, Artist, QualityProfile, RootFolder +import logging +l = logging.getLogger(__name__) + +class LidarrClient: + def __init__(self, base_url: str, api_token: str): + self.base_url = base_url + self.api_token = api_token + self.headers = { + 'X-Api-Key': self.api_token + } + + def _get(self, endpoint: str, params: Optional[dict] = None): + response = requests.get(f"{self.base_url}{endpoint}", headers=self.headers, params=params) + response.raise_for_status() + return response.json() + def _post(self, endpoint: str, json: dict): + response = requests.post(f"{self.base_url}{endpoint}", headers=self.headers, json=json) + response.raise_for_status() + return response.json() + + def _put(self, endpoint: str, json: dict): + response = requests.put(f"{self.base_url}{endpoint}", headers=self.headers, json=json) + response.raise_for_status() + return response.json() + + def get_album(self, album_id: int) -> Album: + l.debug(f"Getting album {album_id}") + data = self._get(f"/api/v1/album/{album_id}") + return Album(**data) + + def get_artist(self, artist_id: int) -> Artist: + l.debug(f"Getting artist {artist_id}") + data = self._get(f"/api/v1/artist/{artist_id}") + return Artist(**data) + + def search(self, term: str) -> List[object]: + l.debug(f"Searching for {term}") + data = self._get("/api/v1/search", params={"term": term}) + results = [] + for item in data: + if 'artist' in item: + results.append(Artist(**item['artist'])) + elif 'album' in item: + results.append(Album(**item['album'])) + return results + # A method which takes a List[object] end external URL as parameter, and returns the object from the List[object] which has the same external URL as the parameter. + def get_object_by_external_url(self, objects: List[object], external_url: str) -> object: + l.debug(f"Getting object by external URL {external_url}") + # We need to check whether the external_url matches intl-[a-zA-Z]{2}\/ it has to be replaced by an empty string + external_url = re.sub(r"intl-[a-zA-Z]{2}\/", "", external_url) + for obj in objects: + # object can either be an Album or an Artist, so it can be verified and casted + if isinstance(obj, Album): + for link in obj.links: + if link['url'] == external_url: + return obj + elif isinstance(obj, Artist): + for link in obj.links: + if link['url'] == external_url: + return obj + + return None + # A method to get all Albums from List[object] where the name equals the parameter name + def get_albums_by_name(self, objects: List[object], name: str) -> List[Album]: + l.debug(f"Getting albums by name {name}") + albums = [] + for obj in objects: + if isinstance(obj, Album) and obj.title == name: + artist = Artist(**obj.artist) + obj.artist = artist + albums.append(obj) + return albums + + # a method to get all artists from List[object] where the name equals the parameter name + def get_artists_by_name(self, objects: List[object], name: str) -> List[Artist]: + l.debug(f"Getting artists by name {name}") + artists = [] + for obj in objects: + if isinstance(obj, Artist) and obj.artistName == name: + artists.append(obj) + return artists + + def create_album(self, album: Album) -> Album: + l.debug(f"Creating album {album.title}") + json_artist = album.artist.__dict__ + album.artist = json_artist + data = self._post("/api/v1/album", json=album.__dict__) + return Album(**data) + + def update_album(self, album_id: int, album: Album) -> Album: + l.debug(f"Updating album {album_id}") + json_artist = album.artist.__dict__ + album.artist = json_artist + data = self._put(f"/api/v1/album/{album_id}", json=album.__dict__) + return Album(**data) + + def create_artist(self, artist: Artist) -> Artist: + l.debug(f"Creating artist {artist.artistName}") + data = self._post("/api/v1/artist", json=artist.__dict__) + return Artist(**data) + + def update_artist(self, artist_id: int, artist: Artist) -> Artist: + l.debug(f"Updating artist {artist_id}") + data = self._put(f"/api/v1/artist/{artist_id}", json=artist.__dict__) + return Artist(**data) + + # shorthand method to set artist to monitored + def monitor_artist(self, artist: Artist): + artist.monitored = True + l.debug(f"Monitoring artist {artist.artistName}") + if artist.id == 0: + artist = self.create_artist(artist) + else: + self.update_artist(artist.id, artist) + # shorthand method to set album to monitored + def monitor_album(self, album: Album): + album.monitored = True + + l.debug(f"Monitoring album {album.title}") + if album.id == 0: + album = self.create_album(album) + else: + self.update_album(album.id, album) + + # a method to query /api/v1/rootfolder and return a List[RootFolder] + def get_root_folders(self) -> List[RootFolder]: + l.debug("Getting root folders") + data = self._get("/api/v1/rootfolder") + return [RootFolder(**folder) for folder in data] + + # a method to query /api/v1/qualityprofile and return a List[QualityProfile] + def get_quality_profiles(self) -> List[QualityProfile]: + l.debug("Getting quality profiles") + data = self._get("/api/v1/qualityprofile") + return [QualityProfile(**profile) for profile in data] + + \ No newline at end of file diff --git a/migrations/versions/d13088ebddc5_add_lidarr_processed_flag.py b/migrations/versions/d13088ebddc5_add_lidarr_processed_flag.py new file mode 100644 index 0000000..b8eb25b --- /dev/null +++ b/migrations/versions/d13088ebddc5_add_lidarr_processed_flag.py @@ -0,0 +1,32 @@ +"""Add lidarr_processed flag + +Revision ID: d13088ebddc5 +Revises: 18d056f49f59 +Create Date: 2024-12-03 22:44:21.287754 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd13088ebddc5' +down_revision = '18d056f49f59' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('track', schema=None) as batch_op: + batch_op.add_column(sa.Column('lidarr_processed', sa.Boolean(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('track', schema=None) as batch_op: + batch_op.drop_column('lidarr_processed') + + # ### end Alembic commands ### diff --git a/templates/admin.html b/templates/admin.html index d0164c4..7c55275 100644 --- a/templates/admin.html +++ b/templates/admin.html @@ -11,6 +11,9 @@
  • Tasks
    • +
  • + Lidarr +
    • diff --git a/templates/admin/lidarr.html b/templates/admin/lidarr.html new file mode 100644 index 0000000..d6a243c --- /dev/null +++ b/templates/admin/lidarr.html @@ -0,0 +1,32 @@ +{% extends "admin.html" %} + +{% block admin_content %} +
    +

    Lidarr Configuration

    + {% if error %} + + {% else %} +
    +
    + + +
    +
    + + +
    + +
    + {% endif %} +
    +{% endblock %} \ No newline at end of file From b9ad5be7bc2f5f365ded31abb6effa19df509102 Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 23:11:50 +0000 Subject: [PATCH 31/42] =?UTF-8?q?Bump=20version:=200.1.6=20=E2=86=92=200.1?= =?UTF-8?q?.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- app/version.py | 2 +- version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 379d3e0..3fb8817 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.6 +current_version = 0.1.7 commit = True tag = True diff --git a/app/version.py b/app/version.py index 0a8da88..f1380ee 100644 --- a/app/version.py +++ b/app/version.py @@ -1 +1 @@ -__version__ = "0.1.6" +__version__ = "0.1.7" diff --git a/version.py b/version.py index 0a8da88..f1380ee 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -__version__ = "0.1.6" +__version__ = "0.1.7" From 9a5adfaa5b20b20501187d0ea35e4d6cc5552385 Mon Sep 17 00:00:00 2001 From: Kamil Date: Tue, 3 Dec 2024 23:20:35 +0000 Subject: [PATCH 32/42] Add SPOTIFY_COOKIE_FILE env var and handle correctly when its missing --- app/__init__.py | 10 +++++++++- config.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index 03e226f..8b769b9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -172,7 +172,15 @@ for name, func in filters.filters.items(): from .providers import SpotifyClient -spotify_client = SpotifyClient('/jellyplist/open.spotify.com_cookies.txt') +if app.config['SPOTIFY_COOKIE_FILE']: + if os.path.exists(app.config['SPOTIFY_COOKIE_FILE']): + spotify_client = SpotifyClient(app.config['SPOTIFY_COOKIE_FILE']) + else: + app.logger.error(f"Cookie file {app.config['SPOTIFY_COOKIE_FILE']} does not exist. Exiting.") + sys.exit(1) +else: + spotify_client = SpotifyClient() + spotify_client.authenticate() from .registry import MusicProviderRegistry MusicProviderRegistry.register_provider(spotify_client) diff --git a/config.py b/config.py index 3eb6027..5098be0 100644 --- a/config.py +++ b/config.py @@ -11,6 +11,7 @@ class Config: JELLYFIN_REQUEST_TIMEOUT = int(os.getenv('JELLYFIN_REQUEST_TIMEOUT','10')) SPOTIFY_CLIENT_ID = os.getenv('SPOTIFY_CLIENT_ID') SPOTIFY_CLIENT_SECRET = os.getenv('SPOTIFY_CLIENT_SECRET') + SPOTIFY_COOKIE_FILE = os.getenv('SPOTIFY_COOKIE_FILE') JELLYPLIST_DB_HOST = os.getenv('JELLYPLIST_DB_HOST') JELLYPLIST_DB_PORT = int(os.getenv('JELLYPLIST_DB_PORT','5432')) JELLYPLIST_DB_USER = os.getenv('JELLYPLIST_DB_USER') From 07503a8003a4d854088fe13d13d6ea20884dfeb4 Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 00:03:29 +0000 Subject: [PATCH 33/42] update path and download state if the track has a jellyfin id set. --- app/tasks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/tasks.py b/app/tasks.py index d9059d2..2bf3356 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -57,6 +57,18 @@ def update_all_playlists_track_status(self): if track.filesystem_path and os.path.exists(track.filesystem_path): available_tracks += 1 track.downloaded = True + #If not found in filesystem, but a jellyfin_id is set, query the jellyfin server for the track and populate the filesystem_path from the response with the path + elif track.jellyfin_id: + jellyfin_track = jellyfin.get_item(jellyfin_admin_token, track.jellyfin_id) + if jellyfin_track and os.path.exists(jellyfin_track['Path']): + track.filesystem_path = jellyfin_track['Path'] + track.downloaded = True + available_tracks += 1 + else: + track.downloaded = False + track.filesystem_path = None + + else: track.downloaded = False From 30ea28ed6e016ea6c767a430c13c7143b07b8aee Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 00:18:34 +0000 Subject: [PATCH 34/42] session commit :[ --- app/tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/tasks.py b/app/tasks.py index 2bf3356..22e26f7 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -55,18 +55,24 @@ def update_all_playlists_track_status(self): for track in playlist.tracks: total_tracks += 1 if track.filesystem_path and os.path.exists(track.filesystem_path): + app.logger.debug(f"Track {track.name} is already downloaded at {track.filesystem_path}.") available_tracks += 1 track.downloaded = True + db.session.commit() #If not found in filesystem, but a jellyfin_id is set, query the jellyfin server for the track and populate the filesystem_path from the response with the path elif track.jellyfin_id: jellyfin_track = jellyfin.get_item(jellyfin_admin_token, track.jellyfin_id) if jellyfin_track and os.path.exists(jellyfin_track['Path']): + app.logger.debug(f"Track {track.name} found in Jellyfin at {jellyfin_track['Path']}.") track.filesystem_path = jellyfin_track['Path'] track.downloaded = True + db.session.commit() available_tracks += 1 else: track.downloaded = False track.filesystem_path = None + db.session.commit() + From e43d36dd24f6bfbdef05e6b80f246d80e238a13a Mon Sep 17 00:00:00 2001 From: Artur Y <10753921+artyorsh@users.noreply.github.com> Date: Wed, 4 Dec 2024 21:52:56 +0100 Subject: [PATCH 35/42] build docker image for amd64 and arm64 --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ccc73db..97e6223 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,6 +37,7 @@ jobs: uses: docker/build-push-action@v4 with: context: . + platforms: linux/amd64,linux/arm64 push: true tags: | ghcr.io/${{ github.repository }}:${{ env.VERSION }} From 86f5bf118ae9a46c5267a4a4b2d463a3798279f6 Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 22:19:44 +0000 Subject: [PATCH 36/42] updated .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c07fcde..7460a67 100644 --- a/.gitignore +++ b/.gitignore @@ -76,4 +76,5 @@ coverage/ *cookies*.txt *.code-workspace set_env.sh -notes.md \ No newline at end of file +notes.md +DEV_BUILD \ No newline at end of file From e2d37b77b0b282eedfef8e9b7d3d9c0d51fdccfd Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 22:21:15 +0000 Subject: [PATCH 37/42] Added: MUSIC_STORAGE_BASE_PATH env variable --- config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 5098be0..0b62eae 100644 --- a/config.py +++ b/config.py @@ -31,13 +31,17 @@ class Config: LIDARR_API_KEY = os.getenv('LIDARR_API_KEY','') LIDARR_URL = os.getenv('LIDARR_URL','') LIDARR_MONITOR_ARTISTS = os.getenv('LIDARR_MONITOR_ARTISTS','false').lower() == 'true' + MUSIC_STORAGE_BASE_PATH = os.getenv('MUSIC_STORAGE_BASE_PATH') # SpotDL specific configuration SPOTDL_CONFIG = { 'cookie_file': '/jellyplist/cookies.txt', - 'output': '/jellyplist_downloads/__jellyplist/{track-id}', + # combine the path provided in MUSIC_STORAGE_BASE_PATH with the following path __jellyplist/{track-id} to get the value for output + 'threads': 12 } + if os.getenv('MUSIC_STORAGE_BASE_PATH'): + SPOTDL_CONFIG['output_file'] = os.path.join(MUSIC_STORAGE_BASE_PATH,'__jellyplist/{track-id}'), @classmethod def validate_env_vars(cls): @@ -52,7 +56,8 @@ class Config: 'JELLYPLIST_DB_HOST' : cls.JELLYPLIST_DB_HOST, 'JELLYPLIST_DB_USER' : cls.JELLYPLIST_DB_USER, 'JELLYPLIST_DB_PASSWORD' : cls.JELLYPLIST_DB_PASSWORD, - 'REDIS_URL': cls.REDIS_URL + 'REDIS_URL': cls.REDIS_URL, + 'MUSIC_STORAGE_BASE_PATH': cls.MUSIC_STORAGE_BASE_PATH } missing_vars = [var for var, value in required_vars.items() if not value] From 1ee0087b8fdbec00dc6658d11f685512e1ec37c3 Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 22:22:04 +0000 Subject: [PATCH 38/42] reworked the celery task management --- app/__init__.py | 4 -- app/functions.py | 42 ------------- app/providers/spotify.py | 7 +++ app/routes/routes.py | 23 +++---- app/tasks.py | 92 +++++++++++++++++++++------- jellyfin/client.py | 6 +- templates/admin/tasks.html | 1 + templates/partials/_task_status.html | 7 +++ 8 files changed, 95 insertions(+), 87 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 8b769b9..ff74384 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -160,10 +160,6 @@ app.logger.debug(f"Debug logging active") from app.routes import pl_bp, routes, jellyfin_routes app.register_blueprint(pl_bp) -from . import tasks -if "worker" in sys.argv: - tasks.release_lock("download_missing_tracks_lock") - from app import filters # Import the filters dictionary # Register all filters diff --git a/app/functions.py b/app/functions.py index c188087..f82805a 100644 --- a/app/functions.py +++ b/app/functions.py @@ -17,48 +17,6 @@ from spotipy.exceptions import SpotifyException import re -TASK_STATUS = { - 'update_all_playlists_track_status': None, - 'download_missing_tracks': None, - 'check_for_playlist_updates': None, - 'update_jellyfin_id_for_downloaded_tracks' : None, - - -} -if app.config['LIDARR_API_KEY']: - TASK_STATUS['request_lidarr'] = None - -LOCK_KEYS = [ - 'update_all_playlists_track_status_lock', - 'download_missing_tracks_lock', - 'check_for_playlist_updates_lock', - 'update_jellyfin_id_for_downloaded_tracks_lock' , - 'full_update_jellyfin_ids', - 'request_lidarr_lock' -] - -def manage_task(task_name): - task_id = TASK_STATUS.get(task_name) - - if task_id: - result = AsyncResult(task_id) - if result.state in ['PENDING', 'STARTED']: - return result.state, result.info if result.info else {} - if task_name == 'update_all_playlists_track_status': - result = tasks.update_all_playlists_track_status.delay() - elif task_name == 'download_missing_tracks': - result = tasks.download_missing_tracks.delay() - elif task_name == 'check_for_playlist_updates': - result = tasks.check_for_playlist_updates.delay() - elif task_name == 'update_jellyfin_id_for_downloaded_tracks': - result = tasks.update_jellyfin_id_for_downloaded_tracks.delay() - elif task_name == 'request_lidarr': - result = tasks.request_lidarr.delay() - - TASK_STATUS[task_name] = result.id - return result.state, result.info if result.info else {} - - def prepPlaylistData(playlist: base.Playlist) -> Optional[CombinedPlaylistData]: jellyfin_user = JellyfinUser.query.filter_by(jellyfin_user_id=session['jellyfin_user_id']).first() if not jellyfin_user: diff --git a/app/providers/spotify.py b/app/providers/spotify.py index 07c8da9..82d4e80 100644 --- a/app/providers/spotify.py +++ b/app/providers/spotify.py @@ -127,6 +127,13 @@ class SpotifyClient(MusicProviderClient): } l.debug(f"starting request: {self.base_url}/{endpoint}") response = requests.get(f"{self.base_url}/{endpoint}", headers=headers, params=params, cookies=self.cookies) + # if the response is unauthorized, we need to reauthenticate + if response.status_code == 401: + l.debug("reauthenticating") + self.authenticate() + headers['authorization'] = f'Bearer {self.session_data.get("accessToken", "")}' + headers['client-token'] = self.client_token.get('token','') + response = requests.get(f"{self.base_url}/{endpoint}", headers=headers, params=params, cookies=self.cookies) response.raise_for_status() return response.json() diff --git a/app/routes/routes.py b/app/routes/routes.py index ef9da3d..c45252c 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -68,14 +68,11 @@ def save_lidarr_config(): @functions.jellyfin_admin_required def task_manager(): statuses = {} - for task_name, task_id in functions.TASK_STATUS.items(): - if task_id: - result = AsyncResult(task_id) - statuses[task_name] = {'state': result.state, 'info': result.info if result.info else {}} - else: - statuses[task_name] = {'state': 'NOT STARTED', 'info': {}} + for task_name, task_id in tasks.task_manager.tasks.items(): + statuses[task_name] = tasks.task_manager.get_task_status(task_name) + - return render_template('admin/tasks.html', tasks=statuses,lock_keys = functions.LOCK_KEYS) + return render_template('admin/tasks.html', tasks=statuses) @app.route('/admin') @app.route('/admin/link_issues') @@ -115,7 +112,7 @@ def link_issues(): @app.route('/run_task/', methods=['POST']) @functions.jellyfin_admin_required def run_task(task_name): - status, info = functions.manage_task(task_name) + status, info = tasks.task_manager.start_task(task_name) # Rendere nur die aktualisierte Zeile der Task task_info = {task_name: {'state': status, 'info': info}} @@ -126,12 +123,9 @@ def run_task(task_name): @functions.jellyfin_admin_required def task_status(): statuses = {} - for task_name, task_id in functions.TASK_STATUS.items(): - if task_id: - result = AsyncResult(task_id) - statuses[task_name] = {'state': result.state, 'info': result.info if result.info else {}} - else: - statuses[task_name] = {'state': 'NOT STARTED', 'info': {}} + for task_name, task_id in tasks.task_manager.tasks.items(): + statuses[task_name] = tasks.task_manager.get_task_status(task_name) + # Render the HTML partial template instead of returning JSON return render_template('partials/_task_status.html', tasks=statuses) @@ -396,5 +390,6 @@ def unlock_key(): @pl_bp.route('/test') def test(): + tasks.update_all_playlists_track_status() return '' \ No newline at end of file diff --git a/app/tasks.py b/app/tasks.py index 22e26f7..4baddb3 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -11,19 +11,13 @@ from app.models import JellyfinUser,Playlist,Track, user_playlists, playlist_tra import os import redis from celery import current_task,signals +from celery.result import AsyncResult from app.providers import base from app.registry.music_provider_registry import MusicProviderRegistry from lidarr.classes import Artist -def acquire_lock(lock_name, expiration=60): - return redis_client.set(lock_name, "locked", ex=expiration, nx=True) -def release_lock(lock_name): - redis_client.delete(lock_name) -def prepare_logger(): - FORMAT = "[%(asctime)s][%(filename)18s:%(lineno)4s - %(funcName)20s() ] %(message)s" - logging.basicConfig(format=FORMAT) @signals.celeryd_init.connect def setup_log_format(sender, conf, **kwargs): @@ -36,7 +30,7 @@ def setup_log_format(sender, conf, **kwargs): def update_all_playlists_track_status(self): lock_key = "update_all_playlists_track_status_lock" - if acquire_lock(lock_key, expiration=600): + if task_manager.acquire_lock(lock_key, expiration=600): try: with app.app_context(): playlists = Playlist.query.all() @@ -51,19 +45,26 @@ def update_all_playlists_track_status(self): for playlist in playlists: total_tracks = 0 available_tracks = 0 - app.logger.debug(f"Current Playlist: {playlist.name} [{playlist.id}:{playlist.provider_playlist_id}]" ) + app.logger.info(f"Current Playlist: {playlist.name} [{playlist.id}:{playlist.provider_playlist_id}]" ) for track in playlist.tracks: total_tracks += 1 + app.logger.debug(f"Processing track: {track.name} [{track.provider_track_id}]") + app.logger.debug(f"\tPath = {track.filesystem_path}") + if track.filesystem_path: + app.logger.debug(f"\tPath exists = {os.path.exists(track.filesystem_path)}") + app.logger.debug(f"\tJellyfinID = {track.jellyfin_id}") if track.filesystem_path and os.path.exists(track.filesystem_path): - app.logger.debug(f"Track {track.name} is already downloaded at {track.filesystem_path}.") + app.logger.info(f"Track {track.name} is already downloaded at {track.filesystem_path}.") available_tracks += 1 track.downloaded = True db.session.commit() #If not found in filesystem, but a jellyfin_id is set, query the jellyfin server for the track and populate the filesystem_path from the response with the path elif track.jellyfin_id: jellyfin_track = jellyfin.get_item(jellyfin_admin_token, track.jellyfin_id) + app.logger.debug(f"\tJellyfin Path: {jellyfin_track['Path']}") + app.logger.debug(f"\tJellyfin Path exists: {os.path.exists(jellyfin_track['Path'])}") if jellyfin_track and os.path.exists(jellyfin_track['Path']): - app.logger.debug(f"Track {track.name} found in Jellyfin at {jellyfin_track['Path']}.") + app.logger.info(f"Track {track.name} found in Jellyfin at {jellyfin_track['Path']}.") track.filesystem_path = jellyfin_track['Path'] track.downloaded = True db.session.commit() @@ -94,7 +95,7 @@ def update_all_playlists_track_status(self): app.logger.info("All playlists' track statuses updated.") return {'status': 'All playlists updated', 'total': total_playlists, 'processed': processed_playlists} finally: - release_lock(lock_key) + task_manager.release_lock(lock_key) else: app.logger.info("Skipping task. Another instance is already running.") return {'status': 'Task skipped, another instance is running'} @@ -104,7 +105,7 @@ def update_all_playlists_track_status(self): def download_missing_tracks(self): lock_key = "download_missing_tracks_lock" - if acquire_lock(lock_key, expiration=1800): + if task_manager.acquire_lock(lock_key, expiration=1800): try: app.logger.info("Starting track download job...") @@ -243,7 +244,7 @@ def download_missing_tracks(self): 'failed': failed_downloads } finally: - release_lock(lock_key) + task_manager.release_lock(lock_key) if app.config['REFRESH_LIBRARIES_AFTER_DOWNLOAD_TASK']: libraries = jellyfin.get_libraries(jellyfin_admin_token) for lib in libraries: @@ -257,7 +258,7 @@ def download_missing_tracks(self): def check_for_playlist_updates(self): lock_key = "check_for_playlist_updates_lock" - if acquire_lock(lock_key, expiration=600): + if task_manager.acquire_lock(lock_key, expiration=600): try: app.logger.info('Starting playlist update check...') with app.app_context(): @@ -355,7 +356,7 @@ def check_for_playlist_updates(self): return {'status': 'Playlist update check completed', 'total': total_playlists, 'processed': processed_playlists} finally: - release_lock(lock_key) + task_manager.release_lock(lock_key) else: app.logger.info("Skipping task. Another instance is already running.") return {'status': 'Task skipped, another instance is running'} @@ -363,15 +364,15 @@ def check_for_playlist_updates(self): @celery.task(bind=True) def update_jellyfin_id_for_downloaded_tracks(self): lock_key = "update_jellyfin_id_for_downloaded_tracks_lock" - full_update_key = 'full_update_jellyfin_ids' - if acquire_lock(lock_key, expiration=600): # Lock for 10 minutes + full_update_key = 'full_update_jellyfin_ids_lock' + if task_manager.acquire_lock(lock_key, expiration=600): # Lock for 10 minutes try: app.logger.info("Starting Jellyfin ID update for tracks...") with app.app_context(): downloaded_tracks = Track.query.filter_by(downloaded=True, jellyfin_id=None).all() - if acquire_lock(full_update_key, expiration=60*60*24): + if task_manager.acquire_lock(full_update_key, expiration=60*60*24): app.logger.info(f"performing full update on jellyfin track ids. (Update tracks and playlists if better quality will be found)") downloaded_tracks = Track.query.all() else: @@ -415,7 +416,7 @@ def update_jellyfin_id_for_downloaded_tracks(self): return {'status': 'All tracks updated', 'total': total_tracks, 'processed': processed_tracks} finally: - release_lock(lock_key) + task_manager.release_lock(lock_key) else: app.logger.info("Skipping task. Another instance is already running.") return {'status': 'Task skipped, another instance is running'} @@ -424,7 +425,7 @@ def update_jellyfin_id_for_downloaded_tracks(self): def request_lidarr(self): lock_key = "request_lidarr_lock" - if acquire_lock(lock_key, expiration=600): + if task_manager.acquire_lock(lock_key, expiration=600): with app.app_context(): if app.config['LIDARR_API_KEY'] and app.config['LIDARR_URL']: from app import lidarr_client @@ -497,11 +498,11 @@ def request_lidarr(self): app.logger.info(f'Requests sent to Lidarr. Total items: {total_items}') return {'status': 'Request sent to Lidarr'} finally: - release_lock(lock_key) + task_manager.release_lock(lock_key) else: app.logger.info('Lidarr API key or URL not set. Skipping request.') - release_lock(lock_key) + task_manager.release_lock(lock_key) else: @@ -594,3 +595,48 @@ def compute_quality_score(result, use_ffprobe=False) -> float: app.logger.warning(f"No valid file path for track {result.get('Name')} - Skipping ffprobe analysis.") return score + + + +class TaskManager: + def __init__(self): + self.tasks = { + 'update_all_playlists_track_status': None, + 'download_missing_tracks': None, + 'check_for_playlist_updates': None, + 'update_jellyfin_id_for_downloaded_tracks': None + } + if app.config['LIDARR_API_KEY']: + self.tasks['request_lidarr'] = None + + def start_task(self, task_name, *args, **kwargs): + if task_name not in self.tasks: + raise ValueError(f"Task {task_name} is not defined.") + task = globals()[task_name].delay(*args, **kwargs) + self.tasks[task_name] = task.id + return task.id + + def get_task_status(self, task_name): + if task_name not in self.tasks: + raise ValueError(f"Task {task_name} is not defined.") + task_id = self.tasks[task_name] + if not task_id: + return {'state': 'NOT STARTED', 'info': {}, 'lock_status': False} + result = AsyncResult(task_id) + lock_status = True if self.get_lock(f"{task_name}_lock") else False + return {'state': result.state, 'info': result.info if result.info else {}, 'lock_status': lock_status} + + def acquire_lock(self, lock_name, expiration=60): + return redis_client.set(lock_name, "locked", ex=expiration, nx=True) + + def release_lock(self, lock_name): + redis_client.delete(lock_name) + + def get_lock(self, lock_name): + return redis_client.get(lock_name) + + def prepare_logger(self): + FORMAT = "[%(asctime)s][%(filename)18s:%(lineno)4s - %(funcName)20s() ] %(message)s" + logging.basicConfig(format=FORMAT) + +task_manager = TaskManager() \ No newline at end of file diff --git a/jellyfin/client.py b/jellyfin/client.py index ae4ab0a..2eb84d7 100644 --- a/jellyfin/client.py +++ b/jellyfin/client.py @@ -310,6 +310,7 @@ class JellyfinClient: response = requests.delete(url, headers=self._get_headers(session_token=session_token), timeout = self.timeout) self.logger.debug(f"Response = {response.status_code}") + logging.getLogger('requests').setLevel(logging.WARNING) if response.status_code == 204: # 204 No Content indicates successful deletion return {"status": "success", "message": "Playlist removed successfully"} @@ -318,11 +319,8 @@ class JellyfinClient: def get_item(self, session_token: str, item_id: str): url = f'{self.base_url}/Items/{item_id}' - self.logger.debug(f"Url={url}") - + logging.getLogger('requests').setLevel(logging.WARNING) response = requests.get(url, headers=self._get_headers(session_token=session_token), timeout = self.timeout) - self.logger.debug(f"Response = {response.status_code}") - if response.status_code == 200: return response.json() else: diff --git a/templates/admin/tasks.html b/templates/admin/tasks.html index d0296be..ee75fdb 100644 --- a/templates/admin/tasks.html +++ b/templates/admin/tasks.html @@ -4,6 +4,7 @@ + diff --git a/templates/partials/_task_status.html b/templates/partials/_task_status.html index 53a02ad..da36087 100644 --- a/templates/partials/_task_status.html +++ b/templates/partials/_task_status.html @@ -1,5 +1,12 @@ {% for task_name, task in tasks.items() %} +
    Locked Task Name Status Progress
    + {% if task.lock_status %} + + {% else %} + + {% endif %} + {{ task_name }} {{ task.state }} From d9dabd0a9ce4c77aabb4260b5c78a59a4145407c Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 22:26:42 +0000 Subject: [PATCH 39/42] test arm build on dev --- .github/workflows/manual-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml index beb7ef2..3f5d82a 100644 --- a/.github/workflows/manual-build.yml +++ b/.github/workflows/manual-build.yml @@ -53,6 +53,7 @@ jobs: uses: docker/build-push-action@v4 with: context: . + platforms: linux/amd64,linux/arm64 push: true tags: | ghcr.io/${{ github.repository }}:${{ github.sha }} From d69ac2299892a59d3ad1759f79554b0d6ff43972 Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 22:58:22 +0000 Subject: [PATCH 40/42] fixed function call --- app/routes/jellyfin_routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/routes/jellyfin_routes.py b/app/routes/jellyfin_routes.py index a781440..6f52970 100644 --- a/app/routes/jellyfin_routes.py +++ b/app/routes/jellyfin_routes.py @@ -5,7 +5,7 @@ from sqlalchemy import insert from app import app, db, jellyfin, functions, device_id,sp from app.models import JellyfinUser, Playlist,Track, playlist_tracks from spotipy.exceptions import SpotifyException - +from app.tasks import task_manager from app.registry.music_provider_registry import MusicProviderRegistry from jellyfin.objects import PlaylistMetadata @@ -73,7 +73,7 @@ def add_playlist(): db.session.add(playlist) db.session.commit() if app.config['START_DOWNLOAD_AFTER_PLAYLIST_ADD']: - functions.manage_task('download_missing_tracks') + task_manager.start_task('download_missing_tracks') # Get the logged-in user user : JellyfinUser = functions._get_logged_in_user() playlist.tracks_available = 0 From 360c4e5b7a4e8a7a852f9213aa8111ee1cd0f8fd Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 23:37:40 +0000 Subject: [PATCH 41/42] changed request-lidarr-schedule to x:50 --- app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index ff74384..bb36dc0 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -83,7 +83,7 @@ def make_celery(app): if app.config['LIDARR_API_KEY']: celery.conf.beat_schedule['request-lidarr-schedule'] = { 'task': 'app.tasks.request_lidarr', - 'schedule': crontab(minute='*/15') + 'schedule': crontab(minute='50') } celery.conf.timezone = 'UTC' From 89a1bc21be24deb311cc8cd287bf30685a5f0faa Mon Sep 17 00:00:00 2001 From: Kamil Date: Wed, 4 Dec 2024 23:38:13 +0000 Subject: [PATCH 42/42] Updated readme and changelog 0.1.7 --- changelogs/0.1.7.md | 59 +++++++++++++++++++++++++++++++++++++++++++++ readme.md | 41 ++++++++++++++++++++++--------- 2 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 changelogs/0.1.7.md diff --git a/changelogs/0.1.7.md b/changelogs/0.1.7.md new file mode 100644 index 0000000..8655237 --- /dev/null +++ b/changelogs/0.1.7.md @@ -0,0 +1,59 @@ +# Whats up in Jellyplist 0.1.6? +### Major overhaul +I´ve been working the past week to make this project work again, after [Spotify announced to deprecate](https://developer.spotify.com/blog/2024-11-27-changes-to-the-web-api) the playlist discover API´s , which were a crucial part of this project. +I also took this opportunity at the same time to do a major overhaul, on how Jellyplist gathers data from a music provider. Music provider API implementations must now implement defined abstract classes to work with Jellyplist, think of it like _plugins_. Jellyplist now, in theory, can gather data from any music provider - just the _plugins_ must be written. It also doesn´t matter, if it have 1,2 or 10 Music Providers to playlists. So stay tuned for more to come. +The next ones will be Deezer and YTM + +After the providers will be implemented, I will at some point do the same with the media backend - so Jellyplist will be able to support other media backends like Navidrome, Plex, Emby and so on... + +### 🆕New API Implementation for Spotify +As mentioned above, I needed a new way to get playlists. +Now, to get them , you don´t need an API Key to authenticate, you even don´t need to be authenticated at all. If you like to have Playlists recommended or created for you, you can use authentication via a cookie. +To do this, add a env var to you `.env` file: +```bash +SPOTIFY_COOKIE_FILE = '/jellyplist/spotify-cookie.txt' +``` +And map the cookie from your local filesystem to the container path you´ve set in the `.env`file +```yaml +... +... + volumes: + - /your/local/path/open.spotify.com_cookies.txt:${SPOTIFY_COOKIE_FILE} +... +... +``` + +### 🆕Lidarr integration is here +To enable the Lidarr integration add these to your `.env` file +```bash +LIDARR_API_KEY = aabbccddeeffgghh11223344 # self explaining +LIDARR_URL = http://:8686 # too +LIDARR_MONITOR_ARTISTS = false # If false, only the corresponding +# album will be set to monitored in lidarr, if true the whole artist +# will be set as monitored. Be careful in the beginning as you might +# hammer your lidarr instance and you indexers. Defaults to false +``` +After you enabled Lidarr integration, make sure to go to _"Admin -> Lidarr"_ and set the default quality profile and music root folder. + +With the Lidarr integration you get a nice workflow: +1. Add Playlist +2. Playlist gets downloaded via SpotDL and is available after some time +3. At some point (every hour on x:50) the requests to Lidarr are made. +4. Lidarr gets all files. +5. Once a day Jellyplist is doing a full update on all tracks, and searches for the same track but with a better quality profile. + +### ⚠️ New required env var +Ensure to add `MUSIC_STORAGE_BASE_PATH` to your `.env` file. +```bash +MUSIC_STORAGE_BASE_PATH = '/storage/media/music' # The base path where +# your music library is located. +# Must be the same value as your music library in jellyfin +``` + +### Other changes, improvements and fixes +- UI/UX: The index page now has content. From there you can directly drop a playlist link +- UI/UX: The Search bar now works with the new API implementation +- UI/UX: A new `Browse All` (per Music Provider) Page from where you can discover playlists +- UI/UX: Check technical details on a track. Just doubleclick a row in the details view of a playlist. +- UI/UX: Allow to link a track even it´s not marked as downloaded. +- UI/UX: Reworked celery task management and the /admin/tasks UI \ No newline at end of file diff --git a/readme.md b/readme.md index cceac52..3ba8886 100644 --- a/readme.md +++ b/readme.md @@ -9,22 +9,22 @@ Jellyplist aims to be a companion app for your self-hosted [Jellyfin](https://je It´s definitely not a general Playlist Manager for Jellyfin. ## Features -- **Discover Playlists**: Use well-known *Featured Playlists* listings. -- **Categories**: Browse playlists by categories +- **Discover Playlists**: Browse playlists like its nothing. - **View Monitored Playlists**: View playlists which are already synced by the server, adding these to your Jellyfin account will make them available immediately - **Search Playlist**: Search for playlists - **No Sign-Up or User-Accounts**: Jellyplist uses your local Jellyfin server for authentication - **Automatically keep track of changes**: Changes in order, added or removed songs will be tracked and synced with Jellyfin. - **Metadata Sync**: Playlist Metadata will be available at your Jellyfin Server +- **Lidarr Integrations**: Automatically submit Artists or only Albums to your Lidarr instance +- **Automatic Quality Upgrades**: When the same track from a playlist is added later with better quality, the playlist in Jellyfin will be updated to use the better sounding track. ## Getting Started The easiest way to start is by using docker and compose. 1. Log in on https://developers.spotify.com/. Go to the dashboard, create an app and get your Client ID and Secret -2. Get your [cookies.txt file for spot-dl ](https://spotdl.readthedocs.io/en/latest/usage/#youtube-music-premium) -> [!IMPORTANT] -> Currently a [youtube premium account](https://spotdl.readthedocs.io/en/latest/usage/#youtube-music-premium) is required, the next release will mitigate this. -3. Prepare a `.env` File +2. Get your [cookies.txt file for spot-dl ](https://spotdl.readthedocs.io/en/latest/usage/#youtube-music-premium) if you want downloaded files to have 256kbit/s, otherwise 128kbit/s +3. Get your cookie-file from open.spotify.com , this works the same way as in step 2. +4. Prepare a `.env` File ``` POSTGRES_USER = jellyplist POSTGRES_PASSWORD = jellyplist @@ -37,10 +37,27 @@ SPOTIFY_CLIENT_SECRET = JELLYPLIST_DB_HOST = postgres-jellyplist #Hostname of the db Container JELLYPLIST_DB_USER = jellyplist JELLYPLIST_DB_PASSWORD = jellyplist -# Optional: +MUSIC_STORAGE_BASE_PATH = '/storage/media/music' # The base path where your music library is located. Must be the same value as your music library in jellyfin + +### Optional: + # SEARCH_JELLYFIN_BEFORE_DOWNLOAD = false # defaults to true, before attempting to do a download with spotDL , the song will be searched first in the local library + # START_DOWNLOAD_AFTER_PLAYLIST_ADD = true # defaults to false, If a new Playlist is added, the Download Task will be scheduled immediately -# + +# FIND_BEST_MATCH_USE_FFPROBE = true # Use ffprobe to gather quality details from a file to calculate quality score. Otherwise jellyplist will use details provided by jellyfin. defaults to false. + +#REFRESH_LIBRARIES_AFTER_DOWNLOAD_TASK = true # jellyplist will trigger a music library update on your Jellyfin server, in case you dont have `Realtime Monitoring` enabled on your Jellyfin library. Defaults to false. + +# LOG_LEVEL = DEBUG # Defaults to INFO + +# SPOTIFY_COOKIE_FILE = '/jellyplist/spotify-cookie.txt' # Not necesarily needed, but if you like to browse your personal recomendations you must provide it so that the new api implementation is able to authenticate + +### Lidarr integration +# LIDARR_API_KEY = aabbccddeeffgghh11223344 # self explaining +# LIDARR_URL = http://:8686 # too +# LIDARR_MONITOR_ARTISTS = false # If false, only the corresponding album will be set to monitored in lidarr, if true the whole artist will be set as monitored. Be careful in the beginning as you might hammer your lidarr instance and you indexers. Defaults to false + ``` @@ -81,8 +98,9 @@ services: - jellyplist-network volumes: # Map Your cookies.txt file to exac - - /your/local/path/cookies.txt:/jellyplist/cookies.txt - - /storage/media/music:/jellyplist_downloads + - /your/local/path/cookies.txt:/jellyplist/cookies.txt # + - /your/local/path/open.spotify.com_cookies.txt:/jellyplist/spotify-cookie.txt + - ${MUSIC_STORAGE_BASE_PATH}:${MUSIC_STORAGE_BASE_PATH} # Jellyplist must be able to access the file paths like they are stored in Jellyfin env_file: - .env @@ -95,7 +113,8 @@ services: volumes: # Map Your cookies.txt file to exac - /your/local/path/cookies.txt:/jellyplist/cookies.txt - - /storage/media/music:/jellyplist_downloads + - /your/local/path/open.spotify.com_cookies.txt:/jellyplist/spotify-cookie.txt + - ${MUSIC_STORAGE_BASE_PATH}:${MUSIC_STORAGE_BASE_PATH} # Jellyplist must be able to access the file paths like they are stored in Jellyfin env_file: - .env depends_on: