From ca0c726d2917b47b887ef588f6bdb0350233360c Mon Sep 17 00:00:00 2001 From: yzqzss Date: Sat, 15 Jun 2024 15:42:49 +0800 Subject: [PATCH] chore: CRLF to LR --- .gitattributes | 1 + .vscode/extensions.json | 14 +- biliarchiver/cli_tools/up_command.py | 176 +++++++------- biliarchiver/cli_tools/utils.py | 46 ++-- biliarchiver/i18n.py | 40 ++-- biliarchiver/locales/.gitignore | 2 +- biliarchiver/rest_api/bilivid.py | 148 ++++++------ biliarchiver/rest_api/main.py | 330 +++++++++++++-------------- biliarchiver/version.py | 2 +- 9 files changed, 380 insertions(+), 379 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..94f480d --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json index ff320c1..120bcf7 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,7 +1,7 @@ -{ - "recommendations": [ - "charliermarsh.ruff", - "tamasfe.even-better-toml", - "esbenp.prettier-vscode" - ] -} +{ + "recommendations": [ + "charliermarsh.ruff", + "tamasfe.even-better-toml", + "esbenp.prettier-vscode" + ] +} diff --git a/biliarchiver/cli_tools/up_command.py b/biliarchiver/cli_tools/up_command.py index b49c094..c567066 100644 --- a/biliarchiver/cli_tools/up_command.py +++ b/biliarchiver/cli_tools/up_command.py @@ -1,88 +1,88 @@ -from io import TextIOWrapper -import click -import os - -from biliarchiver.cli_tools.utils import read_bvids -from biliarchiver.i18n import _ - - -DEFAULT_COLLECTION = "opensource_movies" -""" -开放 collection ，任何人均可上传。 -通过 biliarchiver 上传的 item 会在24小时内被自动转移到 bilibili_videos collection -""" -""" -An open collection. Anyone can upload. -Items uploaded by biliarchiver will be automatically moved to bilibili_videos collection within 24 hours. -""" - -BILIBILI_VIDEOS_COLLECTION = "bilibili_videos" -""" 由 arkiver 管理。bilibili_videos 属于 social-media-video 的子集 """ -""" Managed by arkiver. bilibili_videos is a subset of social-media-video """ - -BILIBILI_VIDEOS_SUB_1_COLLECTION = "bilibili_videos_sub_1" -""" 由 yzqzss 管理。属于 bilibili_videos 的子集 """ -""" Managed by yzqzss. A subset of bilibili_videos """ - - -@click.command(help=click.style(_("上传至互联网档案馆"), fg="cyan")) -@click.option("--bvids", "-i", type=click.STRING, default=None, help=_("bvids 列表的文件路径")) -@click.option( - "--by-storage-home-dir", - "-a", - is_flag=True, - default=False, - help=_("使用 `$storage_home_dir/videos` 目录下的所有视频"), -) -@click.option( - "--update-existing", "-u", is_flag=True, default=False, help=_("更新已存在的 item") -) -@click.option( - "--collection", - "-c", - default=DEFAULT_COLLECTION, - type=click.Choice( - [ - DEFAULT_COLLECTION, - BILIBILI_VIDEOS_COLLECTION, - BILIBILI_VIDEOS_SUB_1_COLLECTION, - ] - ), - help=_("欲上传至的 collection. (非默认值仅限 collection 管理员使用)") - + f" [default: {DEFAULT_COLLECTION}]", -) -@click.option( - "--delete-after-upload", - "-d", - is_flag=True, - default=False, - help=_("上传后删除视频文件"), -) -def up( - bvids: TextIOWrapper, - by_storage_home_dir: bool, - update_existing: bool, - collection: str, - delete_after_upload: bool, -): - from biliarchiver._biliarchiver_upload_bvid import upload_bvid - from biliarchiver.config import config - - ids = [] - - if by_storage_home_dir: - for bvid_with_upper_part in os.listdir(config.storage_home_dir / "videos"): - bvid = bvid_with_upper_part - if "-" in bvid_with_upper_part: - bvid = bvid_with_upper_part.split("-")[0] - ids.append(bvid) - elif bvids: - ids = read_bvids(bvids) - - for id in ids: - upload_bvid( - id, - update_existing=update_existing, - collection=collection, - delete_after_upload=delete_after_upload, - ) +from io import TextIOWrapper +import click +import os + +from biliarchiver.cli_tools.utils import read_bvids +from biliarchiver.i18n import _ + + +DEFAULT_COLLECTION = "opensource_movies" +""" +开放 collection ，任何人均可上传。 +通过 biliarchiver 上传的 item 会在24小时内被自动转移到 bilibili_videos collection +""" +""" +An open collection. Anyone can upload. +Items uploaded by biliarchiver will be automatically moved to bilibili_videos collection within 24 hours. +""" + +BILIBILI_VIDEOS_COLLECTION = "bilibili_videos" +""" 由 arkiver 管理。bilibili_videos 属于 social-media-video 的子集 """ +""" Managed by arkiver. bilibili_videos is a subset of social-media-video """ + +BILIBILI_VIDEOS_SUB_1_COLLECTION = "bilibili_videos_sub_1" +""" 由 yzqzss 管理。属于 bilibili_videos 的子集 """ +""" Managed by yzqzss. A subset of bilibili_videos """ + + +@click.command(help=click.style(_("上传至互联网档案馆"), fg="cyan")) +@click.option("--bvids", "-i", type=click.STRING, default=None, help=_("bvids 列表的文件路径")) +@click.option( + "--by-storage-home-dir", + "-a", + is_flag=True, + default=False, + help=_("使用 `$storage_home_dir/videos` 目录下的所有视频"), +) +@click.option( + "--update-existing", "-u", is_flag=True, default=False, help=_("更新已存在的 item") +) +@click.option( + "--collection", + "-c", + default=DEFAULT_COLLECTION, + type=click.Choice( + [ + DEFAULT_COLLECTION, + BILIBILI_VIDEOS_COLLECTION, + BILIBILI_VIDEOS_SUB_1_COLLECTION, + ] + ), + help=_("欲上传至的 collection. (非默认值仅限 collection 管理员使用)") + + f" [default: {DEFAULT_COLLECTION}]", +) +@click.option( + "--delete-after-upload", + "-d", + is_flag=True, + default=False, + help=_("上传后删除视频文件"), +) +def up( + bvids: TextIOWrapper, + by_storage_home_dir: bool, + update_existing: bool, + collection: str, + delete_after_upload: bool, +): + from biliarchiver._biliarchiver_upload_bvid import upload_bvid + from biliarchiver.config import config + + ids = [] + + if by_storage_home_dir: + for bvid_with_upper_part in os.listdir(config.storage_home_dir / "videos"): + bvid = bvid_with_upper_part + if "-" in bvid_with_upper_part: + bvid = bvid_with_upper_part.split("-")[0] + ids.append(bvid) + elif bvids: + ids = read_bvids(bvids) + + for id in ids: + upload_bvid( + id, + update_existing=update_existing, + collection=collection, + delete_after_upload=delete_after_upload, + ) diff --git a/biliarchiver/cli_tools/utils.py b/biliarchiver/cli_tools/utils.py index 1c47ad1..6ffde3b 100644 --- a/biliarchiver/cli_tools/utils.py +++ b/biliarchiver/cli_tools/utils.py @@ -1,23 +1,23 @@ -from pathlib import Path -from biliarchiver.utils.identifier import is_bvid -from biliarchiver.i18n import _ - - -def read_bvids(bvids: str) -> list[str]: - bvids_list = None - - file = Path(bvids) - if file.exists() and file.is_file(): - with open(file, "r", encoding="utf-8") as f: - bvids_list = f.read().split() - else: - bvids_list = bvids.split() - - del bvids - - for bvid in bvids_list: - assert is_bvid(bvid), _("bvid {} 不合法").format(bvid) - - assert bvids_list is not None and len(bvids_list) > 0, _("bvids 为空") - - return bvids_list +from pathlib import Path +from biliarchiver.utils.identifier import is_bvid +from biliarchiver.i18n import _ + + +def read_bvids(bvids: str) -> list[str]: + bvids_list = None + + file = Path(bvids) + if file.exists() and file.is_file(): + with open(file, "r", encoding="utf-8") as f: + bvids_list = f.read().split() + else: + bvids_list = bvids.split() + + del bvids + + for bvid in bvids_list: + assert is_bvid(bvid), _("bvid {} 不合法").format(bvid) + + assert bvids_list is not None and len(bvids_list) > 0, _("bvids 为空") + + return bvids_list diff --git a/biliarchiver/i18n.py b/biliarchiver/i18n.py index b1eed8d..4156dea 100644 --- a/biliarchiver/i18n.py +++ b/biliarchiver/i18n.py @@ -1,20 +1,20 @@ -import gettext -import locale -from pathlib import Path -import warnings - -default_lang, default_enc = locale.getdefaultlocale() -default_lang = default_lang or "en" -languages = ["en"] if not default_lang.lower().startswith("zh") else ["zh_CN"] -appname = "biliarchiver" - -localedir = Path(__file__).parent / "locales" -if not localedir.exists(): - warnings.warn("Locales directory not found, i18n will not work.", RuntimeWarning) - -i18n = gettext.translation( - appname, localedir=localedir, fallback=True, languages=languages -) - -_ = i18n.gettext -ngettext = i18n.ngettext +import gettext +import locale +from pathlib import Path +import warnings + +default_lang, default_enc = locale.getdefaultlocale() +default_lang = default_lang or "en" +languages = ["en"] if not default_lang.lower().startswith("zh") else ["zh_CN"] +appname = "biliarchiver" + +localedir = Path(__file__).parent / "locales" +if not localedir.exists(): + warnings.warn("Locales directory not found, i18n will not work.", RuntimeWarning) + +i18n = gettext.translation( + appname, localedir=localedir, fallback=True, languages=languages +) + +_ = i18n.gettext +ngettext = i18n.ngettext diff --git a/biliarchiver/locales/.gitignore b/biliarchiver/locales/.gitignore index 6bb0d4c..cd1f2c9 100644 --- a/biliarchiver/locales/.gitignore +++ b/biliarchiver/locales/.gitignore @@ -1 +1 @@ -*.mo +*.mo diff --git a/biliarchiver/rest_api/bilivid.py b/biliarchiver/rest_api/bilivid.py index cc56708..1de1cc0 100644 --- a/biliarchiver/rest_api/bilivid.py +++ b/biliarchiver/rest_api/bilivid.py @@ -1,74 +1,74 @@ -import asyncio -from enum import Enum -import time -from typing import Optional - -class VideoStatus(str, Enum): - pending = "pending" - downloading = "downloading" - uploading = "uploading" - finished = "finished" - failed = "failed" - - -class BiliVideo: - def __init__(self, bvid: str, status: VideoStatus): - if not bvid.startswith("BV"): - bvid = "BV" + bvid - self.added_time = int(time.time()) - self.bvid = bvid - self.status = status - - - def __str__(self) -> str: - return "\t".join([self.bvid, self.status]) - - async def down(self): - from asyncio import subprocess - from shlex import quote - - cmd = ["biliarchiver", "down" ,"-i", quote(self.bvid), "-s", "--disable-version-check"] - - process: Optional[subprocess.Process] = None - try: - process = await asyncio.create_subprocess_exec(*cmd) - retcode = await process.wait() - process = None - except (KeyboardInterrupt, SystemExit, Exception) as e: - if process: - process.terminate() - await process.wait() - print("download terminated:", e) - return -1 - else: - return retcode - finally: - if process: - process.terminate() - await process.wait() - print("download terminated: (finally)") - - async def up(self) -> int: - from asyncio import subprocess - from shlex import quote - - cmd = ["biliarchiver", "up" ,"-i", quote(self.bvid), "-d"] - - process: Optional[subprocess.Process] = None - try: - process = await subprocess.create_subprocess_exec(*cmd) - retcode = await process.wait() - process = None - except (KeyboardInterrupt, SystemExit, Exception) as e: - if process: - process.terminate() - await process.wait() - print("upload terminated", e) - return -1 - else: - return retcode - finally: - if process: - process.terminate() - await process.wait() - print("upload terminated: (finally)") +import asyncio +from enum import Enum +import time +from typing import Optional + +class VideoStatus(str, Enum): + pending = "pending" + downloading = "downloading" + uploading = "uploading" + finished = "finished" + failed = "failed" + + +class BiliVideo: + def __init__(self, bvid: str, status: VideoStatus): + if not bvid.startswith("BV"): + bvid = "BV" + bvid + self.added_time = int(time.time()) + self.bvid = bvid + self.status = status + + + def __str__(self) -> str: + return "\t".join([self.bvid, self.status]) + + async def down(self): + from asyncio import subprocess + from shlex import quote + + cmd = ["biliarchiver", "down" ,"-i", quote(self.bvid), "-s", "--disable-version-check"] + + process: Optional[subprocess.Process] = None + try: + process = await asyncio.create_subprocess_exec(*cmd) + retcode = await process.wait() + process = None + except (KeyboardInterrupt, SystemExit, Exception) as e: + if process: + process.terminate() + await process.wait() + print("download terminated:", e) + return -1 + else: + return retcode + finally: + if process: + process.terminate() + await process.wait() + print("download terminated: (finally)") + + async def up(self) -> int: + from asyncio import subprocess + from shlex import quote + + cmd = ["biliarchiver", "up" ,"-i", quote(self.bvid), "-d"] + + process: Optional[subprocess.Process] = None + try: + process = await subprocess.create_subprocess_exec(*cmd) + retcode = await process.wait() + process = None + except (KeyboardInterrupt, SystemExit, Exception) as e: + if process: + process.terminate() + await process.wait() + print("upload terminated", e) + return -1 + else: + return retcode + finally: + if process: + process.terminate() + await process.wait() + print("upload terminated: (finally)") diff --git a/biliarchiver/rest_api/main.py b/biliarchiver/rest_api/main.py index 0cf7fb9..b6ccf01 100644 --- a/biliarchiver/rest_api/main.py +++ b/biliarchiver/rest_api/main.py @@ -1,165 +1,165 @@ -import asyncio -from asyncio import Queue -from contextlib import asynccontextmanager -from datetime import datetime -from typing import List - -try: - from fastapi import FastAPI -except ImportError: - print("Please install fastapi") - print("`pip install fastapi`") - raise - - -from biliarchiver.rest_api.bilivid import BiliVideo, VideoStatus -from biliarchiver.version import BILI_ARCHIVER_VERSION - - -@asynccontextmanager -async def lifespan(app: FastAPI): - global pending_queue, other_queue - pending_queue = BiliQueue() - other_queue = BiliQueue(maxsize=250) - print("Loading queue...") - load_queue() - print("Queue loaded") - asyncio.create_task(scheduler()) - yield - print("Shutting down...") - save_queue() - print("Queue saved") - - -class BiliQueue(Queue): - def get_all(self) -> List[BiliVideo]: - return list(self._queue) # type: ignore - async def get(self) -> BiliVideo: - return await super().get() - async def remove(self, video: BiliVideo): - try: - self._queue.remove(video) # type: ignore - return True - except ValueError: - return False - def get_nowait(self) -> BiliVideo: - return super().get_nowait() - async def change_status(self, ori_video: BiliVideo, status: VideoStatus): - await self.remove(ori_video) - ori_video.status = status - await self.put(ori_video) - -pending_queue: BiliQueue = None # type: ignore -other_queue: BiliQueue = None # type: ignore - -app = FastAPI(lifespan=lifespan) - -def get_all_items() -> List[BiliVideo]: - l = pending_queue.get_all() + other_queue.get_all() - l.sort(key=lambda x: x.added_time) - return l - -@app.get("/") -async def root(): - return { - "status": "ok", - "biliarchiver": {"version": BILI_ARCHIVER_VERSION}, - "api": {"version": 1}, - "timestamp": int(datetime.now().timestamp()), - } - - -@app.put("/archive/{vid}") -@app.post("/archive/{vid}") -async def add(vid: str): - video = BiliVideo(vid, status=VideoStatus.pending) - await pending_queue.put(video) - return {"success": True, "vid": vid} - - -@app.get("/archive") -async def get_all(): - all_item = get_all_items() - return {"success": True, "total_tasks_queued": pending_queue.qsize(), "items": all_item} - - -@app.get("/archive/{vid}") -async def get_one(vid: str): - all_item = get_all_items() - if vid in [v.bvid for v in all_item]: - v: BiliVideo = [v for v in all_item if v.bvid == vid][0] - return {"success": True, "vid": vid, "status": v.status, "queue_index": all_item.index(v)} - return {"success": False, "vid": vid, "status": "not_found"} # TODO - - -@app.delete("/archive/{vid}") -async def delete(vid: str): - queue_list = pending_queue.get_all() - if vid in [v.bvid for v in queue_list]: - v: BiliVideo = [v for v in queue_list if v.bvid == vid][0] - if await pending_queue.remove(v): - return {"success": True, "vid": vid, "result": "removed", "queue_index": queue_list.index(v)} - - return {"success": False, "vid": vid, "status": "not_found"} - - -async def scheduler(): - while True: - print("Getting a video URL... If no video URL is printed, the queue is empty.") - video = await pending_queue.get() - print(f"Start downloading {video}") - - video.status = VideoStatus.downloading - await other_queue.put(video) - downloaded = False - for _ in range(2): - try: - if retcode := await video.down(): - raise Exception(f"Download failed with retcode {retcode}") - downloaded = True - break - except Exception as e: - print(e) - print(f"(down) Retrying {video}...") - await asyncio.sleep(5) - if not downloaded: - await other_queue.change_status(video, VideoStatus.failed) - print(f"Failed to download {video}") - continue - - print(f"Start uploading {video}") - await other_queue.change_status(video, VideoStatus.uploading) - uploaded = False - for _ in range(3): - try: - retcode = await video.up() - uploaded = True - if retcode != 0: - raise Exception(f"Upload failed with retcode {retcode}") - break - except Exception as e: - print(e) - print(f"(up) Retrying {video}...") - await asyncio.sleep(10) - if not uploaded: - await other_queue.change_status(video, VideoStatus.failed) - print(f"Failed to upload {video}") - continue - - await other_queue.change_status(video, VideoStatus.finished) - print(f"Finished {video}") - -def save_queue(): - with open("queue.txt", "w") as f: - while not pending_queue.empty(): - video = pending_queue.get_nowait() - f.write(str(video) + "\n") - - -def load_queue(): - try: - with open("queue.txt", "r") as f: - while line := f.readline().strip(): - pending_queue.put_nowait(BiliVideo(*line.split("\t"))) - except FileNotFoundError: - pass +import asyncio +from asyncio import Queue +from contextlib import asynccontextmanager +from datetime import datetime +from typing import List + +try: + from fastapi import FastAPI +except ImportError: + print("Please install fastapi") + print("`pip install fastapi`") + raise + + +from biliarchiver.rest_api.bilivid import BiliVideo, VideoStatus +from biliarchiver.version import BILI_ARCHIVER_VERSION + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global pending_queue, other_queue + pending_queue = BiliQueue() + other_queue = BiliQueue(maxsize=250) + print("Loading queue...") + load_queue() + print("Queue loaded") + asyncio.create_task(scheduler()) + yield + print("Shutting down...") + save_queue() + print("Queue saved") + + +class BiliQueue(Queue): + def get_all(self) -> List[BiliVideo]: + return list(self._queue) # type: ignore + async def get(self) -> BiliVideo: + return await super().get() + async def remove(self, video: BiliVideo): + try: + self._queue.remove(video) # type: ignore + return True + except ValueError: + return False + def get_nowait(self) -> BiliVideo: + return super().get_nowait() + async def change_status(self, ori_video: BiliVideo, status: VideoStatus): + await self.remove(ori_video) + ori_video.status = status + await self.put(ori_video) + +pending_queue: BiliQueue = None # type: ignore +other_queue: BiliQueue = None # type: ignore + +app = FastAPI(lifespan=lifespan) + +def get_all_items() -> List[BiliVideo]: + l = pending_queue.get_all() + other_queue.get_all() + l.sort(key=lambda x: x.added_time) + return l + +@app.get("/") +async def root(): + return { + "status": "ok", + "biliarchiver": {"version": BILI_ARCHIVER_VERSION}, + "api": {"version": 1}, + "timestamp": int(datetime.now().timestamp()), + } + + +@app.put("/archive/{vid}") +@app.post("/archive/{vid}") +async def add(vid: str): + video = BiliVideo(vid, status=VideoStatus.pending) + await pending_queue.put(video) + return {"success": True, "vid": vid} + + +@app.get("/archive") +async def get_all(): + all_item = get_all_items() + return {"success": True, "total_tasks_queued": pending_queue.qsize(), "items": all_item} + + +@app.get("/archive/{vid}") +async def get_one(vid: str): + all_item = get_all_items() + if vid in [v.bvid for v in all_item]: + v: BiliVideo = [v for v in all_item if v.bvid == vid][0] + return {"success": True, "vid": vid, "status": v.status, "queue_index": all_item.index(v)} + return {"success": False, "vid": vid, "status": "not_found"} # TODO + + +@app.delete("/archive/{vid}") +async def delete(vid: str): + queue_list = pending_queue.get_all() + if vid in [v.bvid for v in queue_list]: + v: BiliVideo = [v for v in queue_list if v.bvid == vid][0] + if await pending_queue.remove(v): + return {"success": True, "vid": vid, "result": "removed", "queue_index": queue_list.index(v)} + + return {"success": False, "vid": vid, "status": "not_found"} + + +async def scheduler(): + while True: + print("Getting a video URL... If no video URL is printed, the queue is empty.") + video = await pending_queue.get() + print(f"Start downloading {video}") + + video.status = VideoStatus.downloading + await other_queue.put(video) + downloaded = False + for _ in range(2): + try: + if retcode := await video.down(): + raise Exception(f"Download failed with retcode {retcode}") + downloaded = True + break + except Exception as e: + print(e) + print(f"(down) Retrying {video}...") + await asyncio.sleep(5) + if not downloaded: + await other_queue.change_status(video, VideoStatus.failed) + print(f"Failed to download {video}") + continue + + print(f"Start uploading {video}") + await other_queue.change_status(video, VideoStatus.uploading) + uploaded = False + for _ in range(3): + try: + retcode = await video.up() + uploaded = True + if retcode != 0: + raise Exception(f"Upload failed with retcode {retcode}") + break + except Exception as e: + print(e) + print(f"(up) Retrying {video}...") + await asyncio.sleep(10) + if not uploaded: + await other_queue.change_status(video, VideoStatus.failed) + print(f"Failed to upload {video}") + continue + + await other_queue.change_status(video, VideoStatus.finished) + print(f"Finished {video}") + +def save_queue(): + with open("queue.txt", "w") as f: + while not pending_queue.empty(): + video = pending_queue.get_nowait() + f.write(str(video) + "\n") + + +def load_queue(): + try: + with open("queue.txt", "r") as f: + while line := f.readline().strip(): + pending_queue.put_nowait(BiliVideo(*line.split("\t"))) + except FileNotFoundError: + pass diff --git a/biliarchiver/version.py b/biliarchiver/version.py index 1986bd0..bae1c5c 100644 --- a/biliarchiver/version.py +++ b/biliarchiver/version.py @@ -1 +1 @@ -BILI_ARCHIVER_VERSION = "0.1.9" +BILI_ARCHIVER_VERSION = "0.1.9"