From 9f6a40c900827c2e6a64b55da44c51b4845d7b59 Mon Sep 17 00:00:00 2001 From: yiannisha Date: Sat, 30 Oct 2021 15:35:01 +0300 Subject: [PATCH 1/8] Can now install different songs with the same name So basically now in each directory created by zspotify a .song_ids file will be created and appended to every a song is installed in that directory. Now, instead of checking if a file exists by name we're checking with song id. --- zspotify/track.py | 18 ++++++++++++++++-- zspotify/utils.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/zspotify/track.py b/zspotify/track.py index 57960496..2f4daaed 100644 --- a/zspotify/track.py +++ b/zspotify/track.py @@ -11,7 +11,8 @@ from tqdm import tqdm from const import TRACKS, ALBUM, NAME, ITEMS, DISC_NUMBER, TRACK_NUMBER, IS_PLAYABLE, ARTISTS, IMAGES, URL, \ RELEASE_DATE, ID, TRACKS_URL, SAVED_TRACKS_URL, SPLIT_ALBUM_DISCS, ROOT_PATH, DOWNLOAD_FORMAT, CHUNK_SIZE, \ SKIP_EXISTING_FILES, ANTI_BAN_WAIT_TIME, OVERRIDE_AUTO_WAIT, BITRATE, CODEC_MAP, EXT_MAP, DOWNLOAD_REAL_TIME -from utils import sanitize_data, set_audio_tags, set_music_thumbnail, create_download_directory +from utils import sanitize_data, set_audio_tags, set_music_thumbnail, create_download_directory, \ + get_directory_song_ids, add_to_directory_song_ids from zspotify import ZSpotify @@ -74,6 +75,16 @@ def download_track(track_id: str, extra_paths='', prefix=False, prefix_value='', filename = os.path.join( download_directory, f'{song_name}.{EXT_MAP.get(ZSpotify.get_config(DOWNLOAD_FORMAT))}') + check_name = os.path.isfile(filename) and os.path.getsize(filename) + check_id = scraped_song_id in get_directory_song_ids(download_directory) + + # a song with the same name is installed + if not check_id and check_name: + # TODO: count songs with the same name but different id and add prefix based on that + filename = os.path.join( + download_directory, f'{song_name}_1.{EXT_MAP.get(ZSpotify.get_config(DOWNLOAD_FORMAT))}') + + except Exception as e: print('### SKIPPING SONG - FAILED TO QUERY METADATA ###') print(e) @@ -83,7 +94,7 @@ def download_track(track_id: str, extra_paths='', prefix=False, prefix_value='', print('\n### SKIPPING:', song_name, '(SONG IS UNAVAILABLE) ###') else: - if os.path.isfile(filename) and os.path.getsize(filename) and ZSpotify.get_config(SKIP_EXISTING_FILES): + if check_id and ZSpotify.get_config(SKIP_EXISTING_FILES): print('\n### SKIPPING:', song_name, '(SONG ALREADY EXISTS) ###') else: @@ -119,6 +130,9 @@ def download_track(track_id: str, extra_paths='', prefix=False, prefix_value='', release_year, disc_number, track_number) set_music_thumbnail(filename, image_url) + # add song id to download directory's .song_ids file + add_to_directory_song_ids(download_directory, scraped_song_id) + if not ZSpotify.get_config(OVERRIDE_AUTO_WAIT): time.sleep(ZSpotify.get_config(ANTI_BAN_WAIT_TIME)) except Exception as e: diff --git a/zspotify/utils.py b/zspotify/utils.py index b8188d95..e838bbb1 100644 --- a/zspotify/utils.py +++ b/zspotify/utils.py @@ -15,11 +15,38 @@ from const import SANITIZE, ARTIST, TRACKTITLE, ALBUM, YEAR, DISCNUMBER, TRACKNU class MusicFormat(str, Enum): MP3 = 'mp3', OGG = 'ogg', - + def create_download_directory(download_path: str) -> None: + """ Create directory and add a hidden file with song ids """ os.makedirs(download_path, exist_ok=True) + # add hidden file with song ids + hidden_file_path = os.path.join(download_path, '.song_ids') + if not os.path.isfile(hidden_file_path): + with open(hidden_file_path, 'w', encoding='utf-8') as f: + pass + +def get_directory_song_ids(download_path: str) -> list[str]: + """ Gets song ids of songs in directory """ + + song_ids = [] + + hidden_file_path = os.path.join(download_path, '.song_ids') + if os.path.isfile(hidden_file_path): + with open(hidden_file_path, 'r', encoding='utf-8') as file: + song_ids.extend([line.strip() for line in file.readlines()]) + + return song_ids + +def add_to_directory_song_ids(download_path: str, song_id: str) -> None: + """ Appends song_id to .song_ids file in directory """ + + hidden_file_path = os.path.join(download_path, '.song_ids') + # not checking if file exists because we need an exception + # to be raised if something is wrong + with open(hidden_file_path, 'a', encoding='utf-8') as file: + file.write(f'{song_id}\n') def wait(seconds: int = 3) -> None: """ Pause for a set number of seconds """ From 857b1d949a573a33141ea5197c4867a4a9cd50b5 Mon Sep 17 00:00:00 2001 From: yiannisha Date: Sat, 30 Oct 2021 15:42:59 +0300 Subject: [PATCH 2/8] Fixed issue when deleting or moving file --- zspotify/track.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zspotify/track.py b/zspotify/track.py index 2f4daaed..0322f098 100644 --- a/zspotify/track.py +++ b/zspotify/track.py @@ -94,7 +94,7 @@ def download_track(track_id: str, extra_paths='', prefix=False, prefix_value='', print('\n### SKIPPING:', song_name, '(SONG IS UNAVAILABLE) ###') else: - if check_id and ZSpotify.get_config(SKIP_EXISTING_FILES): + if check_id and check_name and ZSpotify.get_config(SKIP_EXISTING_FILES): print('\n### SKIPPING:', song_name, '(SONG ALREADY EXISTS) ###') else: @@ -131,7 +131,8 @@ def download_track(track_id: str, extra_paths='', prefix=False, prefix_value='', set_music_thumbnail(filename, image_url) # add song id to download directory's .song_ids file - add_to_directory_song_ids(download_directory, scraped_song_id) + if not check_id: + add_to_directory_song_ids(download_directory, scraped_song_id) if not ZSpotify.get_config(OVERRIDE_AUTO_WAIT): time.sleep(ZSpotify.get_config(ANTI_BAN_WAIT_TIME)) From b340f92f293f7847c57b1a8e9465e49e5f99c7d7 Mon Sep 17 00:00:00 2001 From: yiannisha Date: Sat, 30 Oct 2021 15:58:09 +0300 Subject: [PATCH 3/8] Added prefixes in case more than two songs have the same name --- zspotify/track.py | 7 +++++-- zspotify/utils.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/zspotify/track.py b/zspotify/track.py index 0322f098..27aac958 100644 --- a/zspotify/track.py +++ b/zspotify/track.py @@ -1,4 +1,5 @@ import os +import re import time from typing import Any, Tuple, List @@ -80,9 +81,11 @@ def download_track(track_id: str, extra_paths='', prefix=False, prefix_value='', # a song with the same name is installed if not check_id and check_name: - # TODO: count songs with the same name but different id and add prefix based on that + c = len([file for file in os.listdir(download_directory) + if re.search(f'^{song_name}_', file)]) + 1 + filename = os.path.join( - download_directory, f'{song_name}_1.{EXT_MAP.get(ZSpotify.get_config(DOWNLOAD_FORMAT))}') + download_directory, f'{song_name}_{c}.{EXT_MAP.get(ZSpotify.get_config(DOWNLOAD_FORMAT))}') except Exception as e: diff --git a/zspotify/utils.py b/zspotify/utils.py index e838bbb1..5f9f5c25 100644 --- a/zspotify/utils.py +++ b/zspotify/utils.py @@ -27,7 +27,7 @@ def create_download_directory(download_path: str) -> None: with open(hidden_file_path, 'w', encoding='utf-8') as f: pass -def get_directory_song_ids(download_path: str) -> list[str]: +def get_directory_song_ids(download_path: str) -> List[str]: """ Gets song ids of songs in directory """ song_ids = [] From 7d355d2ca0fbc911b76d2011a3068dd202a5d963 Mon Sep 17 00:00:00 2001 From: Logykk <35679186+logykk@users.noreply.github.com> Date: Sun, 31 Oct 2021 10:18:48 +1300 Subject: [PATCH 4/8] Fix merge fuckery --- zspotify/track.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/zspotify/track.py b/zspotify/track.py index eebf2432..76fd5f44 100644 --- a/zspotify/track.py +++ b/zspotify/track.py @@ -12,12 +12,8 @@ from tqdm import tqdm from const import TRACKS, ALBUM, NAME, ITEMS, DISC_NUMBER, TRACK_NUMBER, IS_PLAYABLE, ARTISTS, IMAGES, URL, \ RELEASE_DATE, ID, TRACKS_URL, SAVED_TRACKS_URL, SPLIT_ALBUM_DISCS, ROOT_PATH, DOWNLOAD_FORMAT, CHUNK_SIZE, \ SKIP_EXISTING_FILES, ANTI_BAN_WAIT_TIME, OVERRIDE_AUTO_WAIT, BITRATE, CODEC_MAP, EXT_MAP, DOWNLOAD_REAL_TIME -<<<<<<< HEAD -from utils import fix_filename, set_audio_tags, set_music_thumbnail, create_download_directory -======= from utils import sanitize_data, set_audio_tags, set_music_thumbnail, create_download_directory, \ get_directory_song_ids, add_to_directory_song_ids ->>>>>>> fixes#137 from zspotify import ZSpotify From 8053045022fe367ba25c3dd1c4fe76cd40094a16 Mon Sep 17 00:00:00 2001 From: logykk Date: Sun, 31 Oct 2021 10:22:50 +1300 Subject: [PATCH 5/8] fix wrong import --- zspotify/track.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zspotify/track.py b/zspotify/track.py index 76fd5f44..0c04ebab 100644 --- a/zspotify/track.py +++ b/zspotify/track.py @@ -12,7 +12,7 @@ from tqdm import tqdm from const import TRACKS, ALBUM, NAME, ITEMS, DISC_NUMBER, TRACK_NUMBER, IS_PLAYABLE, ARTISTS, IMAGES, URL, \ RELEASE_DATE, ID, TRACKS_URL, SAVED_TRACKS_URL, SPLIT_ALBUM_DISCS, ROOT_PATH, DOWNLOAD_FORMAT, CHUNK_SIZE, \ SKIP_EXISTING_FILES, ANTI_BAN_WAIT_TIME, OVERRIDE_AUTO_WAIT, BITRATE, CODEC_MAP, EXT_MAP, DOWNLOAD_REAL_TIME -from utils import sanitize_data, set_audio_tags, set_music_thumbnail, create_download_directory, \ +from utils import fix_filename, set_audio_tags, set_music_thumbnail, create_download_directory, \ get_directory_song_ids, add_to_directory_song_ids from zspotify import ZSpotify From 3beaa7564cb257743c459a9337d73799de980024 Mon Sep 17 00:00:00 2001 From: Footsiefat <12180913+Footsiefat@users.noreply.github.com> Date: Sun, 31 Oct 2021 13:20:25 +1300 Subject: [PATCH 6/8] added experimental podcast download method --- zspotify/podcast.py | 62 ++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/zspotify/podcast.py b/zspotify/podcast.py index 686ea4ab..f7c337dc 100644 --- a/zspotify/podcast.py +++ b/zspotify/podcast.py @@ -27,7 +27,8 @@ def get_show_episodes(show_id_str) -> list: limit = 50 while True: - resp = ZSpotify.invoke_url_with_params(f'{SHOWS_URL}/{show_id_str}/episodes', limit=limit, offset=offset) + resp = ZSpotify.invoke_url_with_params( + f'{SHOWS_URL}/{show_id_str}/episodes', limit=limit, offset=offset) offset += limit for episode in resp[ITEMS]: episodes.append(episode[ID]) @@ -37,6 +38,33 @@ def get_show_episodes(show_id_str) -> list: return episodes +def download_podcast_experimental(url, filename): + import functools + import pathlib + import shutil + import requests + from tqdm.auto import tqdm + + r = requests.get(url, stream=True, allow_redirects=True) + if r.status_code != 200: + r.raise_for_status() # Will only raise for 4xx codes, so... + raise RuntimeError( + f"Request to {url} returned status code {r.status_code}") + file_size = int(r.headers.get('Content-Length', 0)) + + path = pathlib.Path(filename).expanduser().resolve() + path.parent.mkdir(parents=True, exist_ok=True) + + desc = "(Unknown total file size)" if file_size == 0 else "" + r.raw.read = functools.partial( + r.raw.read, decode_content=True) # Decompress if needed + with tqdm.wrapattr(r.raw, "read", total=file_size, desc=desc) as r_raw: + with path.open("wb") as f: + shutil.copyfileobj(r_raw, f) + + return path + + def download_episode(episode_id) -> None: podcast_name, episode_name = get_episode_info(episode_id) @@ -47,9 +75,6 @@ def download_episode(episode_id) -> None: else: filename = podcast_name + ' - ' + episode_name - episode_id = EpisodeId.from_base62(episode_id) - stream = ZSpotify.get_content_stream(episode_id, ZSpotify.DOWNLOAD_QUALITY) - download_directory = os.path.join( os.path.dirname(__file__), ZSpotify.get_config(ROOT_PODCAST_PATH), @@ -58,33 +83,12 @@ def download_episode(episode_id) -> None: download_directory = os.path.realpath(download_directory) create_download_directory(download_directory) - total_size = stream.input_stream.size + filepath = os.path.join(download_directory, f"{filename}.mp3") - filepath = os.path.join(download_directory, f"{filename}.ogg") - if ( - os.path.isfile(filepath) - and os.path.getsize(filepath) == total_size - and ZSpotify.get_config(SKIP_EXISTING_FILES) - ): - print( - "\n### SKIPPING:", - podcast_name, - "-", - episode_name, - "(EPISODE ALREADY EXISTS) ###", - ) - return + download_url = ZSpotify.invoke_url( + 'https://api-partner.spotify.com/pathfinder/v1/query?operationName=getEpisode&variables={"uri":"spotify:episode:3u0krXMhLcdGZUZkDVmMSt"}&extensions={"persistedQuery":{"version":1,"sha256Hash":"224ba0fd89fcfdfb3a15fa2d82a6112d3f4e2ac88fba5c6713de04d1b72cf482"}}')["data"]["episode"]["audio"]["items"][-1]["url"] - with open(filepath, 'wb') as file, tqdm( - desc=filename, - total=total_size, - unit='B', - unit_scale=True, - unit_divisor=1024 - ) as bar: - for _ in range(int(total_size / ZSpotify.get_config(CHUNK_SIZE)) + 1): - bar.update(file.write( - stream.input_stream.stream().read(ZSpotify.get_config(CHUNK_SIZE)))) + download_podcast_experimental(download_url, filepath) # convert_audio_format(ROOT_PODCAST_PATH + # extra_paths + filename + '.ogg') From 6d7a2b6576ecc6cfa23ba0e2afdd0f1eb5455249 Mon Sep 17 00:00:00 2001 From: Footsiefat <12180913+Footsiefat@users.noreply.github.com> Date: Sun, 31 Oct 2021 13:41:49 +1300 Subject: [PATCH 7/8] 1 --- zspotify/podcast.py | 62 +++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/zspotify/podcast.py b/zspotify/podcast.py index f7c337dc..686ea4ab 100644 --- a/zspotify/podcast.py +++ b/zspotify/podcast.py @@ -27,8 +27,7 @@ def get_show_episodes(show_id_str) -> list: limit = 50 while True: - resp = ZSpotify.invoke_url_with_params( - f'{SHOWS_URL}/{show_id_str}/episodes', limit=limit, offset=offset) + resp = ZSpotify.invoke_url_with_params(f'{SHOWS_URL}/{show_id_str}/episodes', limit=limit, offset=offset) offset += limit for episode in resp[ITEMS]: episodes.append(episode[ID]) @@ -38,33 +37,6 @@ def get_show_episodes(show_id_str) -> list: return episodes -def download_podcast_experimental(url, filename): - import functools - import pathlib - import shutil - import requests - from tqdm.auto import tqdm - - r = requests.get(url, stream=True, allow_redirects=True) - if r.status_code != 200: - r.raise_for_status() # Will only raise for 4xx codes, so... - raise RuntimeError( - f"Request to {url} returned status code {r.status_code}") - file_size = int(r.headers.get('Content-Length', 0)) - - path = pathlib.Path(filename).expanduser().resolve() - path.parent.mkdir(parents=True, exist_ok=True) - - desc = "(Unknown total file size)" if file_size == 0 else "" - r.raw.read = functools.partial( - r.raw.read, decode_content=True) # Decompress if needed - with tqdm.wrapattr(r.raw, "read", total=file_size, desc=desc) as r_raw: - with path.open("wb") as f: - shutil.copyfileobj(r_raw, f) - - return path - - def download_episode(episode_id) -> None: podcast_name, episode_name = get_episode_info(episode_id) @@ -75,6 +47,9 @@ def download_episode(episode_id) -> None: else: filename = podcast_name + ' - ' + episode_name + episode_id = EpisodeId.from_base62(episode_id) + stream = ZSpotify.get_content_stream(episode_id, ZSpotify.DOWNLOAD_QUALITY) + download_directory = os.path.join( os.path.dirname(__file__), ZSpotify.get_config(ROOT_PODCAST_PATH), @@ -83,12 +58,33 @@ def download_episode(episode_id) -> None: download_directory = os.path.realpath(download_directory) create_download_directory(download_directory) - filepath = os.path.join(download_directory, f"{filename}.mp3") + total_size = stream.input_stream.size - download_url = ZSpotify.invoke_url( - 'https://api-partner.spotify.com/pathfinder/v1/query?operationName=getEpisode&variables={"uri":"spotify:episode:3u0krXMhLcdGZUZkDVmMSt"}&extensions={"persistedQuery":{"version":1,"sha256Hash":"224ba0fd89fcfdfb3a15fa2d82a6112d3f4e2ac88fba5c6713de04d1b72cf482"}}')["data"]["episode"]["audio"]["items"][-1]["url"] + filepath = os.path.join(download_directory, f"{filename}.ogg") + if ( + os.path.isfile(filepath) + and os.path.getsize(filepath) == total_size + and ZSpotify.get_config(SKIP_EXISTING_FILES) + ): + print( + "\n### SKIPPING:", + podcast_name, + "-", + episode_name, + "(EPISODE ALREADY EXISTS) ###", + ) + return - download_podcast_experimental(download_url, filepath) + with open(filepath, 'wb') as file, tqdm( + desc=filename, + total=total_size, + unit='B', + unit_scale=True, + unit_divisor=1024 + ) as bar: + for _ in range(int(total_size / ZSpotify.get_config(CHUNK_SIZE)) + 1): + bar.update(file.write( + stream.input_stream.stream().read(ZSpotify.get_config(CHUNK_SIZE)))) # convert_audio_format(ROOT_PODCAST_PATH + # extra_paths + filename + '.ogg') From c5f4bf1ea671e1b76f5203d6609311834c5fee1e Mon Sep 17 00:00:00 2001 From: Footsiefat <12180913+Footsiefat@users.noreply.github.com> Date: Sun, 31 Oct 2021 17:50:27 +1300 Subject: [PATCH 8/8] added proper direct download support for podcasts --- zspotify/podcast.py | 92 +++++++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 28 deletions(-) diff --git a/zspotify/podcast.py b/zspotify/podcast.py index 686ea4ab..265bff7c 100644 --- a/zspotify/podcast.py +++ b/zspotify/podcast.py @@ -27,7 +27,8 @@ def get_show_episodes(show_id_str) -> list: limit = 50 while True: - resp = ZSpotify.invoke_url_with_params(f'{SHOWS_URL}/{show_id_str}/episodes', limit=limit, offset=offset) + resp = ZSpotify.invoke_url_with_params( + f'{SHOWS_URL}/{show_id_str}/episodes', limit=limit, offset=offset) offset += limit for episode in resp[ITEMS]: episodes.append(episode[ID]) @@ -37,6 +38,33 @@ def get_show_episodes(show_id_str) -> list: return episodes +def download_podcast_directly(url, filename): + import functools + import pathlib + import shutil + import requests + from tqdm.auto import tqdm + + r = requests.get(url, stream=True, allow_redirects=True) + if r.status_code != 200: + r.raise_for_status() # Will only raise for 4xx codes, so... + raise RuntimeError( + f"Request to {url} returned status code {r.status_code}") + file_size = int(r.headers.get('Content-Length', 0)) + + path = pathlib.Path(filename).expanduser().resolve() + path.parent.mkdir(parents=True, exist_ok=True) + + desc = "(Unknown total file size)" if file_size == 0 else "" + r.raw.read = functools.partial( + r.raw.read, decode_content=True) # Decompress if needed + with tqdm.wrapattr(r.raw, "read", total=file_size, desc=desc) as r_raw: + with path.open("wb") as f: + shutil.copyfileobj(r_raw, f) + + return path + + def download_episode(episode_id) -> None: podcast_name, episode_name = get_episode_info(episode_id) @@ -47,8 +75,8 @@ def download_episode(episode_id) -> None: else: filename = podcast_name + ' - ' + episode_name - episode_id = EpisodeId.from_base62(episode_id) - stream = ZSpotify.get_content_stream(episode_id, ZSpotify.DOWNLOAD_QUALITY) + direct_download_url = ZSpotify.invoke_url( + 'https://api-partner.spotify.com/pathfinder/v1/query?operationName=getEpisode&variables={"uri":"spotify:episode:' + episode_id + '"}&extensions={"persistedQuery":{"version":1,"sha256Hash":"224ba0fd89fcfdfb3a15fa2d82a6112d3f4e2ac88fba5c6713de04d1b72cf482"}}')["data"]["episode"]["audio"]["items"][-1]["url"] download_directory = os.path.join( os.path.dirname(__file__), @@ -58,33 +86,41 @@ def download_episode(episode_id) -> None: download_directory = os.path.realpath(download_directory) create_download_directory(download_directory) - total_size = stream.input_stream.size + if "anon-podcast.scdn.co" in direct_download_url: + episode_id = EpisodeId.from_base62(episode_id) + stream = ZSpotify.get_content_stream( + episode_id, ZSpotify.DOWNLOAD_QUALITY) - filepath = os.path.join(download_directory, f"{filename}.ogg") - if ( - os.path.isfile(filepath) - and os.path.getsize(filepath) == total_size - and ZSpotify.get_config(SKIP_EXISTING_FILES) - ): - print( - "\n### SKIPPING:", - podcast_name, - "-", - episode_name, - "(EPISODE ALREADY EXISTS) ###", - ) - return + total_size = stream.input_stream.size - with open(filepath, 'wb') as file, tqdm( - desc=filename, - total=total_size, - unit='B', - unit_scale=True, - unit_divisor=1024 - ) as bar: - for _ in range(int(total_size / ZSpotify.get_config(CHUNK_SIZE)) + 1): - bar.update(file.write( - stream.input_stream.stream().read(ZSpotify.get_config(CHUNK_SIZE)))) + filepath = os.path.join(download_directory, f"{filename}.ogg") + if ( + os.path.isfile(filepath) + and os.path.getsize(filepath) == total_size + and ZSpotify.get_config(SKIP_EXISTING_FILES) + ): + print( + "\n### SKIPPING:", + podcast_name, + "-", + episode_name, + "(EPISODE ALREADY EXISTS) ###", + ) + return + + with open(filepath, 'wb') as file, tqdm( + desc=filename, + total=total_size, + unit='B', + unit_scale=True, + unit_divisor=1024 + ) as bar: + for _ in range(int(total_size / ZSpotify.get_config(CHUNK_SIZE)) + 1): + bar.update(file.write( + stream.input_stream.stream().read(ZSpotify.get_config(CHUNK_SIZE)))) + else: + filepath = os.path.join(download_directory, f"{filename}.mp3") + download_podcast_directly(direct_download_url, filepath) # convert_audio_format(ROOT_PODCAST_PATH + # extra_paths + filename + '.ogg')