From 0749f742c31cbcea3aed992c15b146db28310c88 Mon Sep 17 00:00:00 2001 From: Yanis Rigaudeau Date: Sat, 6 May 2023 03:16:26 +0200 Subject: [PATCH] youtube download --- service/youtubeDownloader.py => .drone.yml | 0 .flake8 | 9 ++ .gitignore | 1 + .isort.cfg | 2 + Dockerfile | 0 __main__.py | 32 +++-- cog/misc.py | 6 +- cog/music.py | 24 ++-- config.py | 9 ++ entity/__init__.py | 5 +- entity/album.py | 4 + entity/artist.py | 4 + entity/entry.py | 19 +-- entity/file.py | 3 +- entity/playlist.py | 8 +- entity/queue.py | 8 +- entity/title.py | 4 + framework/__init__.py | 2 + framework/downloader.py | 28 +++++ framework/redis.py | 18 +-- framework/spotify.py | 25 ++++ framework/youtube.py | 74 ++++++++++-- requirements.txt | 9 +- service/__init__.py | 1 + service/fileManager.py | 61 ++++++++++ service/queueManager.py | 4 +- usecase/sources.py | 131 ++++++++++++++++++++- 27 files changed, 417 insertions(+), 74 deletions(-) rename service/youtubeDownloader.py => .drone.yml (100%) create mode 100644 .flake8 create mode 100644 .isort.cfg create mode 100644 Dockerfile create mode 100644 entity/album.py create mode 100644 entity/artist.py create mode 100644 entity/title.py create mode 100644 framework/downloader.py create mode 100644 service/fileManager.py diff --git a/service/youtubeDownloader.py b/.drone.yml similarity index 100% rename from service/youtubeDownloader.py rename to .drone.yml diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..39e94e5 --- /dev/null +++ b/.flake8 @@ -0,0 +1,9 @@ +[flake8] +ignore = + E501, + E731, + W503 +exclude = + .git, + __pycache__, + __init__.py diff --git a/.gitignore b/.gitignore index 5db59f5..9fb9544 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ config.toml +download diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..f238bf7 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile = black diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/__main__.py b/__main__.py index b6f84bf..7ecb3af 100644 --- a/__main__.py +++ b/__main__.py @@ -1,11 +1,13 @@ -from toml import TomlDecodeError -from discord import Bot, Intents +from os import mkdir, path + +from discord import Bot, Intents +from toml import TomlDecodeError -from config import Config -from logger import Logger -from framework import Redis, Youtube from cog import Greetings, Music -from service import QueueManager +from config import Config +from framework import Downloader, Redis, Spotify, Youtube +from logger import Logger +from service import FileManager, QueueManager from usecase import Sources if __name__ == "__main__": @@ -19,11 +21,14 @@ if __name__ == "__main__": print("Config/KeyError : %s" % error) exit(2) + if not path.isdir(config.downloadDirectory): + mkdir(config.downloadDirectory) + # Set Logger logger = Logger(config.logging)() # Redis Client - redis = Redis(logger, config.redis) + redis = Redis(logger, config.redis, config.appName) # Queue Manager queueManager = QueueManager(redis) @@ -32,10 +37,19 @@ if __name__ == "__main__": bot = Bot(intents=Intents.default()) # Youtube Client - youtube = Youtube(bot.loop, config.youtube) + youtube = Youtube(bot.loop, config.youtube, config.downloadDirectory) + + # Spotify Client + spotify = Spotify(bot.loop, config.spotify) + + # Downloader + downloader = Downloader(bot.loop) + + # File Manager + fileManager = FileManager(youtube, downloader, config.downloadDirectory) # Sources - sources = Sources(youtube) + sources = Sources(fileManager, youtube, spotify) # Add Cogs bot.add_cog(Greetings(bot, logger, redis)) diff --git a/cog/misc.py b/cog/misc.py index 2cc5baf..a2929e2 100644 --- a/cog/misc.py +++ b/cog/misc.py @@ -1,7 +1,7 @@ -from discord import Member, Bot, Cog, ApplicationContext -from discord.commands import slash_command from logging import Logger +from discord import ApplicationContext, Bot, Cog, default_permissions, slash_command + from framework.redis import Redis @@ -12,6 +12,7 @@ class Greetings(Cog): self.redis = redis @slash_command() + @default_permissions(administrator=True) async def redis_set(self, context: ApplicationContext, key: str, value: str): self.logger.info(f"redis set {value} at {key}") await self.redis.set(key, value) @@ -19,6 +20,7 @@ class Greetings(Cog): await context.respond(f"redis set {value} at {key}") @slash_command() + @default_permissions(administrator=True) async def redis_get(self, context: ApplicationContext, key: str): self.logger.info(f"redis get {key}") value = await self.redis.get(key) diff --git a/cog/music.py b/cog/music.py index a92a897..33dade5 100644 --- a/cog/music.py +++ b/cog/music.py @@ -1,8 +1,7 @@ -from discord import Bot, Cog, ApplicationContext -from discord.commands import slash_command - -from entity import Entry from logging import Logger + +from discord import ApplicationContext, Bot, Cog, slash_command + from service import QueueManager from usecase import Sources @@ -19,17 +18,8 @@ class Music(Cog): @slash_command(name="play") async def play(self, context: ApplicationContext, query: str): async with self.queueManager(context.guild_id) as queue: - test = await self.sources.processQuery(query) + interaction = await context.respond(f"searching {query} ...") + entries = await self.sources.processQuery(interaction, query) + queue.add(entries) - queue.add( - Entry( - title="title", - artist="artist", - album="album", - thumbnail="thumb", - link="fdsdfsd", - requesterId=context.author.id, - ) - ) - - await context.respond(str(test)) + await interaction.edit_original_response(content=entries[0].title.name) diff --git a/config.py b/config.py index b3f28e3..1ca0478 100644 --- a/config.py +++ b/config.py @@ -13,6 +13,13 @@ class YoutubeConfig: self.region: str = youtube_config["region"] +class SpotifyConfig: + def __init__(self, spotify_config) -> None: + self.clientId: str = spotify_config["clientId"] + self.clientSecret: str = spotify_config["clientSecret"] + self.region: str = spotify_config["region"] + + class LoggingConfig: def __init__(self, logging_config) -> None: self.level: str = logging_config["level"] @@ -29,8 +36,10 @@ class Config: def __init__(self, config_path: str) -> None: self._config = toml.load(config_path) self.appName: str = self._config["appName"] + self.downloadDirectory: str = self._config["downloadDirectory"] self.discord = DiscordConfig(self._config["discord"]) self.youtube = YoutubeConfig(self._config["youtube"]) + self.spotify = SpotifyConfig(self._config["spotify"]) self.logging = LoggingConfig(self._config["logging"]) self.redis = RedisConfig(self._config["redis"]) diff --git a/entity/__init__.py b/entity/__init__.py index 5118729..f95249a 100644 --- a/entity/__init__.py +++ b/entity/__init__.py @@ -1,4 +1,7 @@ -from .queue import Queue +from .album import Album +from .artist import Artist from .entry import Entry from .file import File from .playlist import Playlist +from .queue import Queue +from .title import Title diff --git a/entity/album.py b/entity/album.py new file mode 100644 index 0000000..3587551 --- /dev/null +++ b/entity/album.py @@ -0,0 +1,4 @@ +class Album: + def __init__(self, name: str, url: str) -> None: + self.name = name + self.url = url diff --git a/entity/artist.py b/entity/artist.py new file mode 100644 index 0000000..b165e10 --- /dev/null +++ b/entity/artist.py @@ -0,0 +1,4 @@ +class Artist: + def __init__(self, name: str, url: str) -> None: + self.name = name + self.url = url diff --git a/entity/entry.py b/entity/entry.py index 04f855a..79fcd75 100644 --- a/entity/entry.py +++ b/entity/entry.py @@ -1,24 +1,27 @@ -from .playlist import Playlist +from .album import Album +from .artist import Artist from .file import File +from .playlist import Playlist +from .title import Title class Entry: def __init__( self, - title: str, - artist: str, - album: str, + title: Title, + artist: Artist, + duration: int, thumbnail: str, - link: str, requesterId: int, + album: Album | None = None, playlist: Playlist | None = None, - file: File | None = None, + source: File | str | None = None, ) -> None: self.title = title self.artist = artist self.album = album + self.duration = duration self.thumbnail = thumbnail - self.link = link self.requester = requesterId self.playlist = playlist - self.file = file + self.source = source diff --git a/entity/file.py b/entity/file.py index 6f16528..885300f 100644 --- a/entity/file.py +++ b/entity/file.py @@ -1,5 +1,4 @@ class File: - def __init__(self, name: str, path: str, size: int) -> None: + def __init__(self, name: str, size: int) -> None: self.name = name - self.path = path self.size = size diff --git a/entity/playlist.py b/entity/playlist.py index 454eb49..0651d3d 100644 --- a/entity/playlist.py +++ b/entity/playlist.py @@ -1,4 +1,8 @@ +from .artist import Artist + + class Playlist: - def __init__(self, title: str, url: str) -> None: - self.title = title + def __init__(self, name: str, url: str, owner: Artist) -> None: + self.name = name self.url = url + self.owner = owner diff --git a/entity/queue.py b/entity/queue.py index 0f94f76..b9221bd 100644 --- a/entity/queue.py +++ b/entity/queue.py @@ -1,5 +1,4 @@ from .entry import Entry -from .playlist import Playlist class Queue: @@ -7,11 +6,8 @@ class Queue: self._entries: list[Entry] = [] self.cursor = 0 - def add(self, entry: Entry) -> None: - self._entries.append(entry) - - def addPlalist(self, playlist: Playlist) -> None: - for entry in playlist: + def add(self, entries: list[Entry]) -> None: + for entry in entries: self._entries.append(entry) def remove(self, index: int, recursive: bool) -> None: diff --git a/entity/title.py b/entity/title.py new file mode 100644 index 0000000..142e720 --- /dev/null +++ b/entity/title.py @@ -0,0 +1,4 @@ +class Title: + def __init__(self, name: str, url: str) -> None: + self.name = name + self.url = url diff --git a/framework/__init__.py b/framework/__init__.py index 68f8400..b56f839 100644 --- a/framework/__init__.py +++ b/framework/__init__.py @@ -1,2 +1,4 @@ +from .downloader import Downloader from .redis import Redis +from .spotify import Spotify from .youtube import Youtube diff --git a/framework/downloader.py b/framework/downloader.py new file mode 100644 index 0000000..ad5dc49 --- /dev/null +++ b/framework/downloader.py @@ -0,0 +1,28 @@ +from asyncio import AbstractEventLoop +from os import stat +from urllib.request import urlretrieve + +from entity import File + + +class Downloader: + def __init__(self, loop: AbstractEventLoop) -> None: + self.loop = loop + + def progress(self, progress, block_num: int, block_size: int, total_size: int): + self.loop.create_task(progress(block_num * block_size, total_size)) + + async def download(self, url: str, fileName: str, progress=None) -> File: + await self.loop.run_in_executor( + None, + lambda: urlretrieve( + url, + fileName, + lambda block_num, block_size, total_size: self.progress( + progress, block_num, block_size, total_size + ) + if progress is not None + else None, + ), + ) + return File(name=fileName, size=stat(fileName).st_size) diff --git a/framework/redis.py b/framework/redis.py index 219c299..eaec6ad 100644 --- a/framework/redis.py +++ b/framework/redis.py @@ -1,13 +1,14 @@ -from logging import Logger -from redis.asyncio.lock import Lock -import redis.asyncio as redis import pickle +from logging import Logger + +import redis.asyncio as redis +from redis.asyncio.lock import Lock from config import RedisConfig class Redis: - def __init__(self, logger: Logger, config: RedisConfig) -> None: + def __init__(self, logger: Logger, config: RedisConfig, rootKeyName: str) -> None: self._client = redis.Redis( host=config.host, port=config.port, @@ -15,6 +16,7 @@ class Redis: auto_close_connection_pool=False, ) self._locks: dict[str, Lock] = {} + self.rootKeyName = rootKeyName self.logger = logger async def _get_lock(self, key) -> Lock: @@ -23,23 +25,23 @@ class Redis: return self._locks[key] async def acquire(self, key: str) -> None: - lock = await self._get_lock(f"djembe:queue:{key}") + lock = await self._get_lock(f"{self.rootKeyName}:queue:{key}") await lock.acquire() async def release(self, key: str) -> None: - lock = await self._get_lock(f"djembe:queue:{key}") + lock = await self._get_lock(f"{self.rootKeyName}:queue:{key}") await lock.release() async def get(self, key: str): self.logger.info(f"get value {key} from redis") - value = await self._client.get(f"djembe:queue:{key}") + value = await self._client.get(f"{self.rootKeyName}:queue:{key}") if value: return pickle.loads(value) return None async def set(self, key: str, value) -> None: self.logger.info(f"set value {key} to redis") - await self._client.set(f"djembe:queue:{key}", pickle.dumps(value)) + await self._client.set(f"{self.rootKeyName}:queue:{key}", pickle.dumps(value)) async def close(self) -> None: await self._client.close() diff --git a/framework/spotify.py b/framework/spotify.py index e69de29..8521176 100644 --- a/framework/spotify.py +++ b/framework/spotify.py @@ -0,0 +1,25 @@ +from asyncio import AbstractEventLoop + +import spotipy +from spotipy import MemoryCacheHandler +from spotipy.oauth2 import SpotifyClientCredentials + +from config import SpotifyConfig + + +class Spotify: + def __init__(self, loop: AbstractEventLoop, config: SpotifyConfig) -> None: + self.loop = loop + self.client = spotipy.Spotify( + auth_manager=SpotifyClientCredentials( + client_id=config.clientId, + client_secret=config.clientSecret, + cache_handler=MemoryCacheHandler(), + ), + ) + self.region = config.region + + async def getTrack(self, url: str): + return await self.loop.run_in_executor( + None, lambda: self.client.track(url, self.region) + ) diff --git a/framework/youtube.py b/framework/youtube.py index 6737c12..33611d9 100644 --- a/framework/youtube.py +++ b/framework/youtube.py @@ -1,25 +1,79 @@ -from yt_dlp import YoutubeDL -from youtubesearchpython.__future__ import VideosSearch +import re from asyncio import AbstractEventLoop +from os import path, stat + +from youtubesearchpython.__future__ import VideosSearch +from yt_dlp import YoutubeDL from config import YoutubeConfig +from entity import File class Youtube: - def __init__(self, loop: AbstractEventLoop, config: YoutubeConfig) -> None: - self.params = {} - self.client = YoutubeDL(self.params) + def __init__( + self, loop: AbstractEventLoop, config: YoutubeConfig, downloadDirectory: str + ) -> None: self.loop = loop - self.config = config + self.params = { + "format": "bestaudio/best", + "outtmpl": path.join(downloadDirectory, "%(id)s"), + "restrictfilenames": True, + "noplaylist": True, + "ignoreerrors": True, + "logtostderr": False, + "verbose": False, + "quiet": True, + "no_warnings": True, + "noprogress": True, + } + self.client = YoutubeDL(self.params) + self.language = config.language + self.region = config.region + self.get_id_regex = re.compile( + r"https:\/\/www\.youtube\.com\/watch\?v=(?P[\w-]*)" + ) async def searchVideo(self, query: str): videosSearch = VideosSearch( - query, limit=1, language=self.config.language, region=self.config.region + query, limit=1, language=self.language, region=self.region ) return await videosSearch.next() - async def get_data(self, url: str) -> str: - data = await self.loop.run_in_executor( + async def fetchData(self, url: str): + info = await self.loop.run_in_executor( None, lambda: self.client.extract_info(url, download=False) ) - print(data) + return self.client.sanitize_info(info) + + def getId(self, url: str) -> str | None: + match = self.get_id_regex.search(url) + + if match is None or "id" not in match.groupdict(): + return None + + return match.groupdict()["id"] + + def downloadProgress(self, progress, status): + if status["status"] == "downloading": + self.loop.create_task( + progress( + int(status["downloaded_bytes"]), + int( + status["total_bytes"] + if "total_bytes" in status + else status["total_bytes_estimate"] + ), + ), + ) + + async def download(self, url: str, fileName: str, progress=None) -> None: + if progress is not None: + progress_hook = lambda status: self.downloadProgress(progress, status) + self.client.add_progress_hook(progress_hook) + + await self.loop.run_in_executor(None, lambda: self.client.download(url)) + + if progress is not None: + self.client._progress_hooks.remove(progress_hook) + + return File(name=fileName, size=stat(fileName).st_size) diff --git a/requirements.txt b/requirements.txt index d871e6e..ca1c86c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,11 @@ -py-cord==2.4.1 PyNaCl==1.5.0 -toml==0.10.2 +py-cord==2.4.1 redis==4.5.4 +spotipy==2.23.0 +toml==0.10.2 youtube-search-python==1.6.6 yt-dlp==2023.3.4 + +black==23.3.0 +flake8==6.0.0 +isort==5.12.0 diff --git a/service/__init__.py b/service/__init__.py index 87ca17a..edf3b5d 100644 --- a/service/__init__.py +++ b/service/__init__.py @@ -1 +1,2 @@ +from .fileManager import FileManager from .queueManager import QueueManager diff --git a/service/fileManager.py b/service/fileManager.py new file mode 100644 index 0000000..4b271f3 --- /dev/null +++ b/service/fileManager.py @@ -0,0 +1,61 @@ +import re +from os import path +from time import time + +from discord import Interaction + +from entity import Entry +from framework import Downloader, Youtube + + +class FileManager: + def __init__( + self, youtube: Youtube, downloader: Downloader, downloadDirectory: str + ) -> None: + self.youtube = youtube + self.downloader = downloader + self.downloadDirectory = downloadDirectory + self.youtubeVideoRegex = re.compile( + r"(https:\/\/)?(www|music)\.youtube\.com\/(watch\?v=|shorts\/)\w*" + ) + self.progressLastUpdate = 0 + + def getFileName(self, entryId: str) -> str: + return path.join(self.downloadDirectory, entryId) + + async def downloadProgress( + self, + interaction: Interaction, + entry: Entry, + current_size: int, + total_size: int, + ): + if time() - self.progressLastUpdate > 1: + self.progressLastUpdate = time() + await interaction.edit_original_response( + content=f"Downloading {entry.title.name} [%.2f/%.2f Mo]" + % (current_size / 1024 / 1024, total_size / 1024 / 1024) + ) + + async def download(self, interaction: Interaction, entries: list[Entry]): + for entry in entries: + entryId = self.youtube.getId(entry.title.url) + + fileName = self.getFileName(entryId) + if not path.isfile(fileName) and isinstance(entry.source, str): + if self.youtubeVideoRegex.match(entry.source) is not None: + entry.source = await self.youtube.download( + entry.source, + fileName, + lambda current_size, total_size: self.downloadProgress( + interaction, entry, current_size, total_size + ), + ) + else: + entry.source = await self.downloader.download( + entry.source, + fileName, + lambda current_size, total_size: self.downloadProgress( + interaction, entry, current_size, total_size + ), + ) diff --git a/service/queueManager.py b/service/queueManager.py index 8b53e63..9240362 100644 --- a/service/queueManager.py +++ b/service/queueManager.py @@ -11,11 +11,11 @@ class QueueManager: @asynccontextmanager async def __call__(self, guildId: int) -> Queue: - #await self.acquire(guildId) + # await self.acquire(guildId) queue = await self.get(guildId) yield queue await self.save(guildId) - #await self.release(guildId) + # await self.release(guildId) async def acquire(self, guildId: int) -> None: await self.redis.acquire(guildId) diff --git a/usecase/sources.py b/usecase/sources.py index 8af3bf7..5a89c69 100644 --- a/usecase/sources.py +++ b/usecase/sources.py @@ -1,11 +1,132 @@ +import json import re -from framework import Youtube +from discord import Interaction + +from entity import Album, Artist, Entry, Playlist, Title +from framework import Spotify, Youtube +from service import FileManager class Sources: - def __init__(self, youtube: Youtube) -> None: - self.youtube = youtube + def __init__( + self, fileManager: FileManager, youtube: Youtube, spotify: Spotify + ) -> None: + self.fileManager = fileManager - async def processQuery(self, query: str): - return await self.youtube.searchVideo(query) + # youtube + self.youtube = youtube + self.youtube_video_regex = re.compile( + r"(https:\/\/)?(www|music)\.youtube\.com\/(watch\?v=|shorts\/)\w*" + ) + self.youtube_video_short_regex = re.compile(r"(https:\/\/)?youtu\.be\/\w*") + self.youtube_playlist_regex = re.compile( + r"(https:\/\/)?www\.youtube\.com\/playlist\?list=\w*" + ) + + # spotify + self.spotify = spotify + self.spotify_track_regex = re.compile( + r"(https:\/\/)?(open.spotify.com\/|spotify:)track(\/|:)\w*" + ) + self.spotify_playlist_regex = re.compile( + r"(https:\/\/)?(open.spotify.com\/|spotify:)playlist(\/|:)\w*" + ) + self.spotify_album_regex = re.compile( + r"(https:\/\/)?(open.spotify.com\/|spotify:)album(\/|:)\w*" + ) + self.spotify_artist_regex = re.compile( + r"(https:\/\/)?(open.spotify.com\/|spotify:)artist(\/|:)\w*" + ) + + async def processQuery( + self, interaction: Interaction, query: str + ) -> list[Entry] | None: + if ( + self.youtube_video_regex.match(query) is not None + or self.youtube_video_short_regex.match(query) is not None + ): + data = await self.youtube.fetchData(query) + if data is None: + return None + + with open("search_result.json", "w") as file: + file.write(json.dumps(data)) + + entry = Entry( + title=Title(name=data["title"], url=data["webpage_url"]), + artist=Artist(name=data["channel"], url=data["channel_url"]), + duration=data["duration"], + thumbnail=data["thumbnail"], + requesterId=interaction.user.id, + source=data["webpage_url"], + ) + + await self.fileManager.download(interaction, [entry]) + + return [entry] + + if self.youtube_playlist_regex.match(query) is not None: + data = await self.youtube.getData(query) + if data is None: + return None + + with open("search_result.json", "w") as file: + file.write(json.dumps(data)) + + playlist = Playlist( + title=data["title"], + url=data["webpage_url"], + owner=Artist(name=data["channel"], url=data["channel_url"]), + ) + + return [ + Entry( + title=Title(name=data["title"], url=data["webpage_url"]), + artist=Artist(name=data["channel"], url=data["channel_url"]), + duration=data["duration"], + thumbnail=data["thumbnail"], + requesterId=interaction.user.id, + playlist=playlist, + source=data["url"], + ) + for data in data["entries"] + ] + + if self.spotify_track_regex.match(query) is not None: + data = await self.spotify.getTrack(query) + + with open("spotify_track_result.json", "w") as file: + file.write(json.dumps(data)) + + return [ + Entry( + title=Title( + name=data["name"], url=data["external_urls"]["spotify"] + ), + artist=Artist( + name=data["artists"][0]["name"], + url=data["artists"][0]["external_urls"]["spotify"], + ), + album=Album( + name=data["album"]["name"], + url=data["album"]["external_urls"]["spotify"], + ), + duration=0, + thumbnail=data["album"]["images"][0]["url"], + requesterId=interaction.user.id, + ) + ] + + if ( + self.spotify_playlist_regex.match(query) is not None + or self.spotify_album_regex.match(query) is not None + or self.spotify_artist_regex.match(query) is not None + ): + return + + return await self.processQuery(interaction, await self.search(query)) + + async def search(self, query) -> str: + result = await self.youtube.searchVideo(query) + return result["result"][0]["link"]