From 0143498beea44e2e85be4c80a147da6211a23624 Mon Sep 17 00:00:00 2001 From: yzqzss Date: Sun, 23 Jul 2023 16:38:51 +0800 Subject: [PATCH] =?UTF-8?q?*=20--min-free-space-gb=20=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E7=88=86=E7=9B=98=20*=20=E4=BD=BF=E7=94=A8=20IA=20=E7=9A=84=20?= =?UTF-8?q?check=5Fidentifier.php=20API=20=E5=8A=A0=E5=BF=AB=20item=20?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E6=80=A7=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- biliarchiver/_biliarchiver_upload_bvid.py | 2 +- biliarchiver/archive_bvid.py | 2 +- biliarchiver/cli_tools/bili_archive_bvids.py | 62 ++++++++++++------- .../utils/{string.py => identifier.py} | 0 biliarchiver/utils/storage.py | 21 +++++++ 5 files changed, 62 insertions(+), 25 deletions(-) rename biliarchiver/utils/{string.py => identifier.py} (100%) create mode 100644 biliarchiver/utils/storage.py diff --git a/biliarchiver/_biliarchiver_upload_bvid.py b/biliarchiver/_biliarchiver_upload_bvid.py index 0010151..68b4af9 100644 --- a/biliarchiver/_biliarchiver_upload_bvid.py +++ b/biliarchiver/_biliarchiver_upload_bvid.py @@ -8,7 +8,7 @@ from requests import Response from rich import print from biliarchiver.exception import VideosBasePathNotFoundError -from biliarchiver.utils.string import human_readable_upper_part_map +from biliarchiver.utils.identifier import human_readable_upper_part_map from biliarchiver.config import BILIBILI_IDENTIFIER_PERFIX, config from biliarchiver.utils.dirLock import UploadLock, AlreadyRunningError from biliarchiver.version import BILI_ARCHIVER_VERSION diff --git a/biliarchiver/archive_bvid.py b/biliarchiver/archive_bvid.py index 1dffb81..500ab37 100644 --- a/biliarchiver/archive_bvid.py +++ b/biliarchiver/archive_bvid.py @@ -16,7 +16,7 @@ import json from bilix.sites.bilibili.downloader import DownloaderBilibili from biliarchiver.config import BILIBILI_IDENTIFIER_PERFIX from biliarchiver.config import config -from biliarchiver.utils.string import human_readable_upper_part_map +from biliarchiver.utils.identifier import human_readable_upper_part_map @raise_api_error async def new_get_subtitle_info(client: httpx.AsyncClient, bvid, cid): diff --git a/biliarchiver/cli_tools/bili_archive_bvids.py b/biliarchiver/cli_tools/bili_archive_bvids.py index 4fe1eed..6bbf147 100644 --- a/biliarchiver/cli_tools/bili_archive_bvids.py +++ b/biliarchiver/cli_tools/bili_archive_bvids.py @@ -2,24 +2,22 @@ import asyncio import os import argparse from pathlib import Path -import sys from typing import Optional, Union -from internetarchive import get_item - from biliarchiver.archive_bvid import archive_bvid from biliarchiver.config import config from bilix.sites.bilibili.downloader import DownloaderBilibili from rich.console import Console -from httpx import AsyncClient, Client +from httpx import AsyncClient, Client, TransportError from rich.traceback import install from biliarchiver.utils.http_patch import HttpOnlyCookie_Handler from biliarchiver.utils.version_check import check_outdated_version +from biliarchiver.utils.storage import get_free_space from biliarchiver.version import BILI_ARCHIVER_VERSION install() -from biliarchiver.utils.string import human_readable_upper_part_map +from biliarchiver.utils.identifier import human_readable_upper_part_map from biliarchiver.utils.ffmpeg import check_ffmpeg from biliarchiver.config import BILIBILI_IDENTIFIER_PERFIX @@ -31,6 +29,7 @@ class Args: bvids: str skip_ia: bool from_browser: Optional[str] + min_free_space_gb: int def parse_args(): @@ -39,6 +38,7 @@ def parse_args(): parser.add_argument('-s', '--skip-ia-check', dest='skip_ia', action='store_true', help='不检查 IA 上是否已存在对应 BVID 的 item ，直接开始下载') parser.add_argument('--fb', '--from-browser', dest='from_browser', type=str, help='从指定浏览器导入 cookies (否则导入 config.json 中的 cookies_file) [default: None]', default=None) + parser.add_argument('--min-free-space-gb', dest='min_free_space_gb', type=int, help='最小剩余空间 (GB)，用超退出 [default: 10]', default=10) args = Args(**vars(parser.parse_args())) @@ -48,34 +48,39 @@ def check_ia_item_exist(client: Client, identifier: str) -> bool: cache_dir = config.storage_home_dir / 'ia_item_exist_cache' # check_ia_item_exist_from_cache_file: if (cache_dir / f'{identifier}.mark').exists(): + # print('from cached .mark') return True - cache_dir.mkdir(parents=True, exist_ok=True) def create_item_exist_cache_file(identifier: str) -> Path: with open(cache_dir / f'{identifier}.mark', 'w', encoding='utf-8') as f: f.write('') return cache_dir / f'{identifier}.mark' - # params = { - # 'identifier': identifier, - # 'output': 'json', - # } - # r = client.get('https://archive.org/services/check_identifier.php' ,params=params) - # r.raise_for_status() - # r_json = r.json() - # assert r_json['type'] =='success' - # if r_json['code'] == 'available': - # return False - # elif r_json['code'] == 'not_available': - # return True - # else: - # raise ValueError(f'Unexpected code: {r_json["code"]}') - item = get_item(identifier) - if item.exists: + params = { + 'identifier': identifier, + 'output': 'json', + } + # check_identifier.php API 响应快 + r = None + for _ in range(3): + try: + r = client.get('https://archive.org/services/check_identifier.php', params=params) + break + except TransportError as e: + print(e, 'retrying...') + assert r is not None + r.raise_for_status() + r_json = r.json() + assert r_json['type'] =='success' + if r_json['code'] == 'available': + return False + elif r_json['code'] == 'not_available': # exists + cache_dir.mkdir(parents=True, exist_ok=True) create_item_exist_cache_file(identifier) return True + else: + raise ValueError(f'Unexpected code: {r_json["code"]}') - return False def _main(): args = parse_args() @@ -106,8 +111,16 @@ def _main(): if not logined: return + def check_free_space(): + if args.min_free_space_gb != 0: + if get_free_space(path=config.storage_home_dir) // 1024 // 1024 // 1024 <= args.min_free_space_gb: + return False # not pass + return True # pass + d.progress.start() for index, bvid in enumerate(bvids_from_file): + if not check_free_space(): + break if not args.skip_ia: upper_part = human_readable_upper_part_map(string=bvid, backward=True) remote_identifier = f'{BILIBILI_IDENTIFIER_PERFIX}-{bvid}_p1-{upper_part}' @@ -122,6 +135,9 @@ def _main(): task = loop.create_task(archive_bvid(d, bvid, logined=logined)) + if not check_free_space(): + print(f'剩余空间不足 {args.min_free_space_gb} GiB，退出中...') + while len(asyncio.all_tasks(loop)) > 0: loop.run_until_complete(asyncio.sleep(1)) diff --git a/biliarchiver/utils/string.py b/biliarchiver/utils/identifier.py similarity index 100% rename from biliarchiver/utils/string.py rename to biliarchiver/utils/identifier.py diff --git a/biliarchiver/utils/storage.py b/biliarchiver/utils/storage.py new file mode 100644 index 0000000..05544c2 --- /dev/null +++ b/biliarchiver/utils/storage.py @@ -0,0 +1,21 @@ + +import ctypes +import os +import platform +from typing import Union +from pathlib import Path + + +def get_free_space(path: Union[Path,str]) -> int: + """Return folder/drive free space (in bytes).""" + if not isinstance(path, Path): + path = Path(path) + if not os.path.exists(path): + raise FileNotFoundError(f'{path} File Not Found') + if platform.system() == 'Windows': + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes)) # type: ignore + return free_bytes.value + else: + st = os.statvfs(path) + return st.f_bavail * st.f_frsize