mirror of
https://github.com/saveweb/biliarchiver.git
synced 2024-09-18 18:45:28 -07:00
chore: CRLF to LR
This commit is contained in:
parent
729f0353bf
commit
ca0c726d29
1
.gitattributes
vendored
Normal file
1
.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
* text=auto eol=lf
|
||||
14
.vscode/extensions.json
vendored
14
.vscode/extensions.json
vendored
@ -1,7 +1,7 @@
|
||||
{
|
||||
"recommendations": [
|
||||
"charliermarsh.ruff",
|
||||
"tamasfe.even-better-toml",
|
||||
"esbenp.prettier-vscode"
|
||||
]
|
||||
}
|
||||
{
|
||||
"recommendations": [
|
||||
"charliermarsh.ruff",
|
||||
"tamasfe.even-better-toml",
|
||||
"esbenp.prettier-vscode"
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,88 +1,88 @@
|
||||
from io import TextIOWrapper
|
||||
import click
|
||||
import os
|
||||
|
||||
from biliarchiver.cli_tools.utils import read_bvids
|
||||
from biliarchiver.i18n import _
|
||||
|
||||
|
||||
DEFAULT_COLLECTION = "opensource_movies"
|
||||
"""
|
||||
开放 collection ，任何人均可上传。
|
||||
通过 biliarchiver 上传的 item 会在24小时内被自动转移到 bilibili_videos collection
|
||||
"""
|
||||
"""
|
||||
An open collection. Anyone can upload.
|
||||
Items uploaded by biliarchiver will be automatically moved to bilibili_videos collection within 24 hours.
|
||||
"""
|
||||
|
||||
BILIBILI_VIDEOS_COLLECTION = "bilibili_videos"
|
||||
""" 由 arkiver 管理。bilibili_videos 属于 social-media-video 的子集 """
|
||||
""" Managed by arkiver. bilibili_videos is a subset of social-media-video """
|
||||
|
||||
BILIBILI_VIDEOS_SUB_1_COLLECTION = "bilibili_videos_sub_1"
|
||||
""" 由 yzqzss 管理。属于 bilibili_videos 的子集 """
|
||||
""" Managed by yzqzss. A subset of bilibili_videos """
|
||||
|
||||
|
||||
@click.command(help=click.style(_("上传至互联网档案馆"), fg="cyan"))
|
||||
@click.option("--bvids", "-i", type=click.STRING, default=None, help=_("bvids 列表的文件路径"))
|
||||
@click.option(
|
||||
"--by-storage-home-dir",
|
||||
"-a",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help=_("使用 `$storage_home_dir/videos` 目录下的所有视频"),
|
||||
)
|
||||
@click.option(
|
||||
"--update-existing", "-u", is_flag=True, default=False, help=_("更新已存在的 item")
|
||||
)
|
||||
@click.option(
|
||||
"--collection",
|
||||
"-c",
|
||||
default=DEFAULT_COLLECTION,
|
||||
type=click.Choice(
|
||||
[
|
||||
DEFAULT_COLLECTION,
|
||||
BILIBILI_VIDEOS_COLLECTION,
|
||||
BILIBILI_VIDEOS_SUB_1_COLLECTION,
|
||||
]
|
||||
),
|
||||
help=_("欲上传至的 collection. (非默认值仅限 collection 管理员使用)")
|
||||
+ f" [default: {DEFAULT_COLLECTION}]",
|
||||
)
|
||||
@click.option(
|
||||
"--delete-after-upload",
|
||||
"-d",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help=_("上传后删除视频文件"),
|
||||
)
|
||||
def up(
|
||||
bvids: TextIOWrapper,
|
||||
by_storage_home_dir: bool,
|
||||
update_existing: bool,
|
||||
collection: str,
|
||||
delete_after_upload: bool,
|
||||
):
|
||||
from biliarchiver._biliarchiver_upload_bvid import upload_bvid
|
||||
from biliarchiver.config import config
|
||||
|
||||
ids = []
|
||||
|
||||
if by_storage_home_dir:
|
||||
for bvid_with_upper_part in os.listdir(config.storage_home_dir / "videos"):
|
||||
bvid = bvid_with_upper_part
|
||||
if "-" in bvid_with_upper_part:
|
||||
bvid = bvid_with_upper_part.split("-")[0]
|
||||
ids.append(bvid)
|
||||
elif bvids:
|
||||
ids = read_bvids(bvids)
|
||||
|
||||
for id in ids:
|
||||
upload_bvid(
|
||||
id,
|
||||
update_existing=update_existing,
|
||||
collection=collection,
|
||||
delete_after_upload=delete_after_upload,
|
||||
)
|
||||
from io import TextIOWrapper
|
||||
import click
|
||||
import os
|
||||
|
||||
from biliarchiver.cli_tools.utils import read_bvids
|
||||
from biliarchiver.i18n import _
|
||||
|
||||
|
||||
DEFAULT_COLLECTION = "opensource_movies"
|
||||
"""
|
||||
开放 collection ，任何人均可上传。
|
||||
通过 biliarchiver 上传的 item 会在24小时内被自动转移到 bilibili_videos collection
|
||||
"""
|
||||
"""
|
||||
An open collection. Anyone can upload.
|
||||
Items uploaded by biliarchiver will be automatically moved to bilibili_videos collection within 24 hours.
|
||||
"""
|
||||
|
||||
BILIBILI_VIDEOS_COLLECTION = "bilibili_videos"
|
||||
""" 由 arkiver 管理。bilibili_videos 属于 social-media-video 的子集 """
|
||||
""" Managed by arkiver. bilibili_videos is a subset of social-media-video """
|
||||
|
||||
BILIBILI_VIDEOS_SUB_1_COLLECTION = "bilibili_videos_sub_1"
|
||||
""" 由 yzqzss 管理。属于 bilibili_videos 的子集 """
|
||||
""" Managed by yzqzss. A subset of bilibili_videos """
|
||||
|
||||
|
||||
@click.command(help=click.style(_("上传至互联网档案馆"), fg="cyan"))
|
||||
@click.option("--bvids", "-i", type=click.STRING, default=None, help=_("bvids 列表的文件路径"))
|
||||
@click.option(
|
||||
"--by-storage-home-dir",
|
||||
"-a",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help=_("使用 `$storage_home_dir/videos` 目录下的所有视频"),
|
||||
)
|
||||
@click.option(
|
||||
"--update-existing", "-u", is_flag=True, default=False, help=_("更新已存在的 item")
|
||||
)
|
||||
@click.option(
|
||||
"--collection",
|
||||
"-c",
|
||||
default=DEFAULT_COLLECTION,
|
||||
type=click.Choice(
|
||||
[
|
||||
DEFAULT_COLLECTION,
|
||||
BILIBILI_VIDEOS_COLLECTION,
|
||||
BILIBILI_VIDEOS_SUB_1_COLLECTION,
|
||||
]
|
||||
),
|
||||
help=_("欲上传至的 collection. (非默认值仅限 collection 管理员使用)")
|
||||
+ f" [default: {DEFAULT_COLLECTION}]",
|
||||
)
|
||||
@click.option(
|
||||
"--delete-after-upload",
|
||||
"-d",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help=_("上传后删除视频文件"),
|
||||
)
|
||||
def up(
|
||||
bvids: TextIOWrapper,
|
||||
by_storage_home_dir: bool,
|
||||
update_existing: bool,
|
||||
collection: str,
|
||||
delete_after_upload: bool,
|
||||
):
|
||||
from biliarchiver._biliarchiver_upload_bvid import upload_bvid
|
||||
from biliarchiver.config import config
|
||||
|
||||
ids = []
|
||||
|
||||
if by_storage_home_dir:
|
||||
for bvid_with_upper_part in os.listdir(config.storage_home_dir / "videos"):
|
||||
bvid = bvid_with_upper_part
|
||||
if "-" in bvid_with_upper_part:
|
||||
bvid = bvid_with_upper_part.split("-")[0]
|
||||
ids.append(bvid)
|
||||
elif bvids:
|
||||
ids = read_bvids(bvids)
|
||||
|
||||
for id in ids:
|
||||
upload_bvid(
|
||||
id,
|
||||
update_existing=update_existing,
|
||||
collection=collection,
|
||||
delete_after_upload=delete_after_upload,
|
||||
)
|
||||
|
||||
@ -1,23 +1,23 @@
|
||||
from pathlib import Path
|
||||
from biliarchiver.utils.identifier import is_bvid
|
||||
from biliarchiver.i18n import _
|
||||
|
||||
|
||||
def read_bvids(bvids: str) -> list[str]:
|
||||
bvids_list = None
|
||||
|
||||
file = Path(bvids)
|
||||
if file.exists() and file.is_file():
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
bvids_list = f.read().split()
|
||||
else:
|
||||
bvids_list = bvids.split()
|
||||
|
||||
del bvids
|
||||
|
||||
for bvid in bvids_list:
|
||||
assert is_bvid(bvid), _("bvid {} 不合法").format(bvid)
|
||||
|
||||
assert bvids_list is not None and len(bvids_list) > 0, _("bvids 为空")
|
||||
|
||||
return bvids_list
|
||||
from pathlib import Path
|
||||
from biliarchiver.utils.identifier import is_bvid
|
||||
from biliarchiver.i18n import _
|
||||
|
||||
|
||||
def read_bvids(bvids: str) -> list[str]:
|
||||
bvids_list = None
|
||||
|
||||
file = Path(bvids)
|
||||
if file.exists() and file.is_file():
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
bvids_list = f.read().split()
|
||||
else:
|
||||
bvids_list = bvids.split()
|
||||
|
||||
del bvids
|
||||
|
||||
for bvid in bvids_list:
|
||||
assert is_bvid(bvid), _("bvid {} 不合法").format(bvid)
|
||||
|
||||
assert bvids_list is not None and len(bvids_list) > 0, _("bvids 为空")
|
||||
|
||||
return bvids_list
|
||||
|
||||
@ -1,20 +1,20 @@
|
||||
import gettext
|
||||
import locale
|
||||
from pathlib import Path
|
||||
import warnings
|
||||
|
||||
default_lang, default_enc = locale.getdefaultlocale()
|
||||
default_lang = default_lang or "en"
|
||||
languages = ["en"] if not default_lang.lower().startswith("zh") else ["zh_CN"]
|
||||
appname = "biliarchiver"
|
||||
|
||||
localedir = Path(__file__).parent / "locales"
|
||||
if not localedir.exists():
|
||||
warnings.warn("Locales directory not found, i18n will not work.", RuntimeWarning)
|
||||
|
||||
i18n = gettext.translation(
|
||||
appname, localedir=localedir, fallback=True, languages=languages
|
||||
)
|
||||
|
||||
_ = i18n.gettext
|
||||
ngettext = i18n.ngettext
|
||||
import gettext
|
||||
import locale
|
||||
from pathlib import Path
|
||||
import warnings
|
||||
|
||||
default_lang, default_enc = locale.getdefaultlocale()
|
||||
default_lang = default_lang or "en"
|
||||
languages = ["en"] if not default_lang.lower().startswith("zh") else ["zh_CN"]
|
||||
appname = "biliarchiver"
|
||||
|
||||
localedir = Path(__file__).parent / "locales"
|
||||
if not localedir.exists():
|
||||
warnings.warn("Locales directory not found, i18n will not work.", RuntimeWarning)
|
||||
|
||||
i18n = gettext.translation(
|
||||
appname, localedir=localedir, fallback=True, languages=languages
|
||||
)
|
||||
|
||||
_ = i18n.gettext
|
||||
ngettext = i18n.ngettext
|
||||
|
||||
2
biliarchiver/locales/.gitignore
vendored
2
biliarchiver/locales/.gitignore
vendored
@ -1 +1 @@
|
||||
*.mo
|
||||
*.mo
|
||||
|
||||
@ -1,74 +1,74 @@
|
||||
import asyncio
|
||||
from enum import Enum
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
class VideoStatus(str, Enum):
|
||||
pending = "pending"
|
||||
downloading = "downloading"
|
||||
uploading = "uploading"
|
||||
finished = "finished"
|
||||
failed = "failed"
|
||||
|
||||
|
||||
class BiliVideo:
|
||||
def __init__(self, bvid: str, status: VideoStatus):
|
||||
if not bvid.startswith("BV"):
|
||||
bvid = "BV" + bvid
|
||||
self.added_time = int(time.time())
|
||||
self.bvid = bvid
|
||||
self.status = status
|
||||
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "\t".join([self.bvid, self.status])
|
||||
|
||||
async def down(self):
|
||||
from asyncio import subprocess
|
||||
from shlex import quote
|
||||
|
||||
cmd = ["biliarchiver", "down" ,"-i", quote(self.bvid), "-s", "--disable-version-check"]
|
||||
|
||||
process: Optional[subprocess.Process] = None
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(*cmd)
|
||||
retcode = await process.wait()
|
||||
process = None
|
||||
except (KeyboardInterrupt, SystemExit, Exception) as e:
|
||||
if process:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
print("download terminated:", e)
|
||||
return -1
|
||||
else:
|
||||
return retcode
|
||||
finally:
|
||||
if process:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
print("download terminated: (finally)")
|
||||
|
||||
async def up(self) -> int:
|
||||
from asyncio import subprocess
|
||||
from shlex import quote
|
||||
|
||||
cmd = ["biliarchiver", "up" ,"-i", quote(self.bvid), "-d"]
|
||||
|
||||
process: Optional[subprocess.Process] = None
|
||||
try:
|
||||
process = await subprocess.create_subprocess_exec(*cmd)
|
||||
retcode = await process.wait()
|
||||
process = None
|
||||
except (KeyboardInterrupt, SystemExit, Exception) as e:
|
||||
if process:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
print("upload terminated", e)
|
||||
return -1
|
||||
else:
|
||||
return retcode
|
||||
finally:
|
||||
if process:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
print("upload terminated: (finally)")
|
||||
import asyncio
|
||||
from enum import Enum
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
class VideoStatus(str, Enum):
|
||||
pending = "pending"
|
||||
downloading = "downloading"
|
||||
uploading = "uploading"
|
||||
finished = "finished"
|
||||
failed = "failed"
|
||||
|
||||
|
||||
class BiliVideo:
|
||||
def __init__(self, bvid: str, status: VideoStatus):
|
||||
if not bvid.startswith("BV"):
|
||||
bvid = "BV" + bvid
|
||||
self.added_time = int(time.time())
|
||||
self.bvid = bvid
|
||||
self.status = status
|
||||
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "\t".join([self.bvid, self.status])
|
||||
|
||||
async def down(self):
|
||||
from asyncio import subprocess
|
||||
from shlex import quote
|
||||
|
||||
cmd = ["biliarchiver", "down" ,"-i", quote(self.bvid), "-s", "--disable-version-check"]
|
||||
|
||||
process: Optional[subprocess.Process] = None
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(*cmd)
|
||||
retcode = await process.wait()
|
||||
process = None
|
||||
except (KeyboardInterrupt, SystemExit, Exception) as e:
|
||||
if process:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
print("download terminated:", e)
|
||||
return -1
|
||||
else:
|
||||
return retcode
|
||||
finally:
|
||||
if process:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
print("download terminated: (finally)")
|
||||
|
||||
async def up(self) -> int:
|
||||
from asyncio import subprocess
|
||||
from shlex import quote
|
||||
|
||||
cmd = ["biliarchiver", "up" ,"-i", quote(self.bvid), "-d"]
|
||||
|
||||
process: Optional[subprocess.Process] = None
|
||||
try:
|
||||
process = await subprocess.create_subprocess_exec(*cmd)
|
||||
retcode = await process.wait()
|
||||
process = None
|
||||
except (KeyboardInterrupt, SystemExit, Exception) as e:
|
||||
if process:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
print("upload terminated", e)
|
||||
return -1
|
||||
else:
|
||||
return retcode
|
||||
finally:
|
||||
if process:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
print("upload terminated: (finally)")
|
||||
|
||||
@ -1,165 +1,165 @@
|
||||
import asyncio
|
||||
from asyncio import Queue
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
try:
|
||||
from fastapi import FastAPI
|
||||
except ImportError:
|
||||
print("Please install fastapi")
|
||||
print("`pip install fastapi`")
|
||||
raise
|
||||
|
||||
|
||||
from biliarchiver.rest_api.bilivid import BiliVideo, VideoStatus
|
||||
from biliarchiver.version import BILI_ARCHIVER_VERSION
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
global pending_queue, other_queue
|
||||
pending_queue = BiliQueue()
|
||||
other_queue = BiliQueue(maxsize=250)
|
||||
print("Loading queue...")
|
||||
load_queue()
|
||||
print("Queue loaded")
|
||||
asyncio.create_task(scheduler())
|
||||
yield
|
||||
print("Shutting down...")
|
||||
save_queue()
|
||||
print("Queue saved")
|
||||
|
||||
|
||||
class BiliQueue(Queue):
|
||||
def get_all(self) -> List[BiliVideo]:
|
||||
return list(self._queue) # type: ignore
|
||||
async def get(self) -> BiliVideo:
|
||||
return await super().get()
|
||||
async def remove(self, video: BiliVideo):
|
||||
try:
|
||||
self._queue.remove(video) # type: ignore
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
def get_nowait(self) -> BiliVideo:
|
||||
return super().get_nowait()
|
||||
async def change_status(self, ori_video: BiliVideo, status: VideoStatus):
|
||||
await self.remove(ori_video)
|
||||
ori_video.status = status
|
||||
await self.put(ori_video)
|
||||
|
||||
pending_queue: BiliQueue = None # type: ignore
|
||||
other_queue: BiliQueue = None # type: ignore
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
def get_all_items() -> List[BiliVideo]:
|
||||
l = pending_queue.get_all() + other_queue.get_all()
|
||||
l.sort(key=lambda x: x.added_time)
|
||||
return l
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
return {
|
||||
"status": "ok",
|
||||
"biliarchiver": {"version": BILI_ARCHIVER_VERSION},
|
||||
"api": {"version": 1},
|
||||
"timestamp": int(datetime.now().timestamp()),
|
||||
}
|
||||
|
||||
|
||||
@app.put("/archive/{vid}")
|
||||
@app.post("/archive/{vid}")
|
||||
async def add(vid: str):
|
||||
video = BiliVideo(vid, status=VideoStatus.pending)
|
||||
await pending_queue.put(video)
|
||||
return {"success": True, "vid": vid}
|
||||
|
||||
|
||||
@app.get("/archive")
|
||||
async def get_all():
|
||||
all_item = get_all_items()
|
||||
return {"success": True, "total_tasks_queued": pending_queue.qsize(), "items": all_item}
|
||||
|
||||
|
||||
@app.get("/archive/{vid}")
|
||||
async def get_one(vid: str):
|
||||
all_item = get_all_items()
|
||||
if vid in [v.bvid for v in all_item]:
|
||||
v: BiliVideo = [v for v in all_item if v.bvid == vid][0]
|
||||
return {"success": True, "vid": vid, "status": v.status, "queue_index": all_item.index(v)}
|
||||
return {"success": False, "vid": vid, "status": "not_found"} # TODO
|
||||
|
||||
|
||||
@app.delete("/archive/{vid}")
|
||||
async def delete(vid: str):
|
||||
queue_list = pending_queue.get_all()
|
||||
if vid in [v.bvid for v in queue_list]:
|
||||
v: BiliVideo = [v for v in queue_list if v.bvid == vid][0]
|
||||
if await pending_queue.remove(v):
|
||||
return {"success": True, "vid": vid, "result": "removed", "queue_index": queue_list.index(v)}
|
||||
|
||||
return {"success": False, "vid": vid, "status": "not_found"}
|
||||
|
||||
|
||||
async def scheduler():
|
||||
while True:
|
||||
print("Getting a video URL... If no video URL is printed, the queue is empty.")
|
||||
video = await pending_queue.get()
|
||||
print(f"Start downloading {video}")
|
||||
|
||||
video.status = VideoStatus.downloading
|
||||
await other_queue.put(video)
|
||||
downloaded = False
|
||||
for _ in range(2):
|
||||
try:
|
||||
if retcode := await video.down():
|
||||
raise Exception(f"Download failed with retcode {retcode}")
|
||||
downloaded = True
|
||||
break
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(f"(down) Retrying {video}...")
|
||||
await asyncio.sleep(5)
|
||||
if not downloaded:
|
||||
await other_queue.change_status(video, VideoStatus.failed)
|
||||
print(f"Failed to download {video}")
|
||||
continue
|
||||
|
||||
print(f"Start uploading {video}")
|
||||
await other_queue.change_status(video, VideoStatus.uploading)
|
||||
uploaded = False
|
||||
for _ in range(3):
|
||||
try:
|
||||
retcode = await video.up()
|
||||
uploaded = True
|
||||
if retcode != 0:
|
||||
raise Exception(f"Upload failed with retcode {retcode}")
|
||||
break
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(f"(up) Retrying {video}...")
|
||||
await asyncio.sleep(10)
|
||||
if not uploaded:
|
||||
await other_queue.change_status(video, VideoStatus.failed)
|
||||
print(f"Failed to upload {video}")
|
||||
continue
|
||||
|
||||
await other_queue.change_status(video, VideoStatus.finished)
|
||||
print(f"Finished {video}")
|
||||
|
||||
def save_queue():
|
||||
with open("queue.txt", "w") as f:
|
||||
while not pending_queue.empty():
|
||||
video = pending_queue.get_nowait()
|
||||
f.write(str(video) + "\n")
|
||||
|
||||
|
||||
def load_queue():
|
||||
try:
|
||||
with open("queue.txt", "r") as f:
|
||||
while line := f.readline().strip():
|
||||
pending_queue.put_nowait(BiliVideo(*line.split("\t")))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
import asyncio
|
||||
from asyncio import Queue
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
try:
|
||||
from fastapi import FastAPI
|
||||
except ImportError:
|
||||
print("Please install fastapi")
|
||||
print("`pip install fastapi`")
|
||||
raise
|
||||
|
||||
|
||||
from biliarchiver.rest_api.bilivid import BiliVideo, VideoStatus
|
||||
from biliarchiver.version import BILI_ARCHIVER_VERSION
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
global pending_queue, other_queue
|
||||
pending_queue = BiliQueue()
|
||||
other_queue = BiliQueue(maxsize=250)
|
||||
print("Loading queue...")
|
||||
load_queue()
|
||||
print("Queue loaded")
|
||||
asyncio.create_task(scheduler())
|
||||
yield
|
||||
print("Shutting down...")
|
||||
save_queue()
|
||||
print("Queue saved")
|
||||
|
||||
|
||||
class BiliQueue(Queue):
|
||||
def get_all(self) -> List[BiliVideo]:
|
||||
return list(self._queue) # type: ignore
|
||||
async def get(self) -> BiliVideo:
|
||||
return await super().get()
|
||||
async def remove(self, video: BiliVideo):
|
||||
try:
|
||||
self._queue.remove(video) # type: ignore
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
def get_nowait(self) -> BiliVideo:
|
||||
return super().get_nowait()
|
||||
async def change_status(self, ori_video: BiliVideo, status: VideoStatus):
|
||||
await self.remove(ori_video)
|
||||
ori_video.status = status
|
||||
await self.put(ori_video)
|
||||
|
||||
pending_queue: BiliQueue = None # type: ignore
|
||||
other_queue: BiliQueue = None # type: ignore
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
def get_all_items() -> List[BiliVideo]:
|
||||
l = pending_queue.get_all() + other_queue.get_all()
|
||||
l.sort(key=lambda x: x.added_time)
|
||||
return l
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
return {
|
||||
"status": "ok",
|
||||
"biliarchiver": {"version": BILI_ARCHIVER_VERSION},
|
||||
"api": {"version": 1},
|
||||
"timestamp": int(datetime.now().timestamp()),
|
||||
}
|
||||
|
||||
|
||||
@app.put("/archive/{vid}")
|
||||
@app.post("/archive/{vid}")
|
||||
async def add(vid: str):
|
||||
video = BiliVideo(vid, status=VideoStatus.pending)
|
||||
await pending_queue.put(video)
|
||||
return {"success": True, "vid": vid}
|
||||
|
||||
|
||||
@app.get("/archive")
|
||||
async def get_all():
|
||||
all_item = get_all_items()
|
||||
return {"success": True, "total_tasks_queued": pending_queue.qsize(), "items": all_item}
|
||||
|
||||
|
||||
@app.get("/archive/{vid}")
|
||||
async def get_one(vid: str):
|
||||
all_item = get_all_items()
|
||||
if vid in [v.bvid for v in all_item]:
|
||||
v: BiliVideo = [v for v in all_item if v.bvid == vid][0]
|
||||
return {"success": True, "vid": vid, "status": v.status, "queue_index": all_item.index(v)}
|
||||
return {"success": False, "vid": vid, "status": "not_found"} # TODO
|
||||
|
||||
|
||||
@app.delete("/archive/{vid}")
|
||||
async def delete(vid: str):
|
||||
queue_list = pending_queue.get_all()
|
||||
if vid in [v.bvid for v in queue_list]:
|
||||
v: BiliVideo = [v for v in queue_list if v.bvid == vid][0]
|
||||
if await pending_queue.remove(v):
|
||||
return {"success": True, "vid": vid, "result": "removed", "queue_index": queue_list.index(v)}
|
||||
|
||||
return {"success": False, "vid": vid, "status": "not_found"}
|
||||
|
||||
|
||||
async def scheduler():
|
||||
while True:
|
||||
print("Getting a video URL... If no video URL is printed, the queue is empty.")
|
||||
video = await pending_queue.get()
|
||||
print(f"Start downloading {video}")
|
||||
|
||||
video.status = VideoStatus.downloading
|
||||
await other_queue.put(video)
|
||||
downloaded = False
|
||||
for _ in range(2):
|
||||
try:
|
||||
if retcode := await video.down():
|
||||
raise Exception(f"Download failed with retcode {retcode}")
|
||||
downloaded = True
|
||||
break
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(f"(down) Retrying {video}...")
|
||||
await asyncio.sleep(5)
|
||||
if not downloaded:
|
||||
await other_queue.change_status(video, VideoStatus.failed)
|
||||
print(f"Failed to download {video}")
|
||||
continue
|
||||
|
||||
print(f"Start uploading {video}")
|
||||
await other_queue.change_status(video, VideoStatus.uploading)
|
||||
uploaded = False
|
||||
for _ in range(3):
|
||||
try:
|
||||
retcode = await video.up()
|
||||
uploaded = True
|
||||
if retcode != 0:
|
||||
raise Exception(f"Upload failed with retcode {retcode}")
|
||||
break
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(f"(up) Retrying {video}...")
|
||||
await asyncio.sleep(10)
|
||||
if not uploaded:
|
||||
await other_queue.change_status(video, VideoStatus.failed)
|
||||
print(f"Failed to upload {video}")
|
||||
continue
|
||||
|
||||
await other_queue.change_status(video, VideoStatus.finished)
|
||||
print(f"Finished {video}")
|
||||
|
||||
def save_queue():
|
||||
with open("queue.txt", "w") as f:
|
||||
while not pending_queue.empty():
|
||||
video = pending_queue.get_nowait()
|
||||
f.write(str(video) + "\n")
|
||||
|
||||
|
||||
def load_queue():
|
||||
try:
|
||||
with open("queue.txt", "r") as f:
|
||||
while line := f.readline().strip():
|
||||
pending_queue.put_nowait(BiliVideo(*line.split("\t")))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
@ -1 +1 @@
|
||||
BILI_ARCHIVER_VERSION = "0.1.9"
|
||||
BILI_ARCHIVER_VERSION = "0.1.9"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user