hevc

mediatype
pid find

.gitignore

@@ -3,4 +3,5 @@ bvids/
 sess_data.txt
 biliup.home
 .venv/
-__pycache__/
+__pycache__/
+videos/

_biliup_archive_bvid.py

@@ -37,11 +37,10 @@ api.get_subtitle_info = new_get_subtitle_info
 
 
 async def archive_bvid(d: DownloaderBilibili, bvid: str):
-    if not os.path.exists('biliup.home'):
-        raise Exception('先创建 biliup.home 文件')
-    # 需要先实例化一个用来进行http请求的client
-    # d = DownloaderBilibili(video_concurrency=5, part_concurrency=10, hierarchy=True, sess_data=sess_data)
-    # first we should initialize a http client
+    assert d.hierarchy is True, 'hierarchy 必须为 True' # 为了保持后续目录结构、文件命名的一致性
+    assert d.client.cookies.get('SESSDATA') is not None, 'sess_data 不能为空' # 开个大会员呗，能下 4k 呢。
+    assert os.path.exists('biliup.home'), '先创建 biliup.home 文件' # 防误操作
+
     url = f'https://www.bilibili.com/video/{bvid}/'
     # data = await api.get_video_info(client, "https://www.bilibili.com/video/BV1jK4y1N7ST?p=5")
 
@@ -52,13 +51,14 @@ async def archive_bvid(d: DownloaderBilibili, bvid: str):
     os.makedirs(videos_basepath, exist_ok=True)
 
 
-    async with aiofiles.open(f'{videos_basepath}/videos_info.json', 'w', encoding='utf-8') as f:
-        await f.write(json.dumps(videos_info.dict(), ensure_ascii=False, indent=4))
+    # async with aiofiles.open(f'{videos_basepath}/_videos_info.json', 'w', encoding='utf-8') as f:
+    #     # 用于 debug 的来自 bilix 输出的视频信息，包含用户敏感信息（mid 等）
+    #     await f.write(json.dumps(videos_info.dict(), ensure_ascii=False, indent=4))
 
     pid = 0
     for page in videos_info.pages:
         pid += 1
-        if not page.p_name.startswith(f'P{pid}-'):
+        if not page.p_url.endswith(f'?p={pid}'):
             print(f'{bvid} 的第 {pid}P 不存在')
             continue
 
@@ -77,10 +77,17 @@ async def archive_bvid(d: DownloaderBilibili, bvid: str):
         old_h1_title = video_info.h1_title
     
         video_info.pages[video_info.p].p_name = file_basename
-        video_info.h1_title = 'title' * 30 # 超长标题，用来 fallback 到 file_basename
-        cor1 = d.get_video(page.p_url ,video_info=video_info, quality=0,
-                      dm=True, image=True, subtitle=True, path=video_basepath)
+        video_info.h1_title = 'tttttt' * 50 # 假装超长标题，强制 bilix fallback 到 file_basename 作为文件名
+        cor1 = d.get_video(page.p_url ,video_info=video_info, path=video_basepath,
+                    # hevc 优先
+                    quality=0, codec='hev',
+                    # 下载 ass 弹幕(bilix 会自动调用 danmukuC 将 pb 弹幕转为 ass)、封面、字幕
+                    # 他们会被放进 extra 子目录里
+                    dm=True, image=True, subtitle=True
+                    )
+        # 下载原始的 pb 弹幕
         cor2 = d.get_dm(page.p_url, video_info=video_info, path=video_extrapath)
+        # 获取视频详细信息
         cor3 = download_bilibili_video_detail(d.client, bvid, f'{video_extrapath}/{file_basename}.info.json')
         await asyncio.gather(cor1, cor2, cor3)
 
@@ -95,6 +102,7 @@ async def archive_bvid(d: DownloaderBilibili, bvid: str):
 
 async def download_bilibili_video_detail(client, bvid, filename):
     if os.path.exists(filename):
+        print(f'{bvid} 视频详情已存在')
         return
     # url = 'https://api.bilibili.com/x/web-interface/view'
     url = 'https://api.bilibili.com/x/web-interface/view/detail' # 超详细
@@ -105,6 +113,7 @@ async def download_bilibili_video_detail(client, bvid, filename):
     async with aiofiles.open(filename, 'w', encoding='utf-8') as f:
         # f.write(json.dumps(r.json(), indent=4, ensure_ascii=False))
         await f.write(r.text)
+    print(f'{bvid} 视频详情已保存')
 
 # d = DownloaderBilibili(video_concurrency=2, part_concurrency=1, hierarchy=True, sess_data=None)
 # d.progress.start()

_biliup_upload_bvid.py

@@ -17,8 +17,9 @@ def upload_bvid(bvid):
         if os.path.exists(f'{videos_basepath}/{identifier}/_uploaded.mark'):
             print(f'{identifier} 已经上传过了(_uploaded.mark)')
             continue
-        pid = identifier.split('_')[-1][1:]
-        file_basename = identifier[len(identifier_perfix)+1:]
+        if identifier.startswith('_') :
+            print(f'跳过 {identifier}')
+            continue
         if not identifier.startswith(identifier_perfix):
             print(f'{identifier} 不是以 {identifier_perfix} 开头的正确 identifier')
             continue
@@ -26,6 +27,9 @@ def upload_bvid(bvid):
             print(f'{identifier} 没有下载完成')
             continue
 
+        pid = identifier.split('_')[-1][1:]
+        file_basename = identifier[len(identifier_perfix)+1:]
+    
         print(f'开始上传 {identifier}')
         item = get_item(identifier)
         if item.exists:
@@ -58,8 +62,8 @@ def upload_bvid(bvid):
 
         with open(f'{videos_basepath}/{identifier}/extra/{file_basename}.info.json', 'r', encoding='utf-8') as f:
             bv_info = json.load(f)
-        with open(f'{videos_basepath}/videos_info.json', 'r', encoding='utf-8') as f:
-            videos_info = json.load(f)
+        # with open(f'{videos_basepath}/_videos_info.json', 'r', encoding='utf-8') as f:
+        #     videos_info = json.load(f)
 
         tags = ['BiliBili', 'video']
         for tag in bv_info['data']['Tags']:
@@ -72,7 +76,7 @@ def upload_bvid(bvid):
                 break
         
         md = {
-            "mediatype": "web",
+            "mediatype": "movies",
             "collection": 'opensource_movies',
             "title": bv_info['data']['View']['title'] + f' P{pid} ' + part ,
             "description": bv_info['data']['View']['desc'],

biliup_archive_daily_bvids.py

@@ -1,12 +1,19 @@
 import asyncio
 import datetime
+import logging
 import os
 import sys
-from bilix.sites.bilibili.downloader import DownloaderBilibili
-from _biliup_archive_bvid import archive_bvid
 import argparse
+
+from _biliup_archive_bvid import archive_bvid
+
+from bilix.sites.bilibili.downloader import DownloaderBilibili
+from rich.console import Console
 import uvloop
 
+from rich.traceback import install
+install()
+
 
 def parse_args():
     parser = argparse.ArgumentParser()
@@ -18,34 +25,40 @@ def parse_args():
 
 def main():
     args = parse_args()
+    print(args.sess_data)
     with open(args.bvids, 'r', encoding='utf-8') as f:
         bvids = f.read().splitlines()
-    async def do():
-        d = DownloaderBilibili(video_concurrency=2, part_concurrency=1, hierarchy=True, sess_data=args.sess_data)
-        d.progress.start()
-        futs = []
-        for bvid in bvids:
-            cor = asyncio.create_task(archive_bvid(d=d, bvid=bvid))
-            fut = asyncio.gather(cor)
-            futs.append(fut)
-            if len(futs) == 2:
-                await asyncio.gather(*futs)
-                futs = []
-        if len(futs) > 0:
-                await asyncio.gather(*futs)
-                futs = []
-        d.progress.stop()
-        await d.aclose()
-    # asyncio.run(do())
-    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
-    loop = asyncio.get_event_loop_policy().get_event_loop()
-    loop.run_until_complete(do())
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+
+    from tasks_limit import tasks_limit
+
+    d = DownloaderBilibili(video_concurrency=tasks_limit, part_concurrency=1, hierarchy=True, sess_data=args.sess_data,
+    )
+    d.progress.start()
+    for bvid in bvids:
+        # 限制同时下载的数量
+        while len(asyncio.all_tasks(loop)) > tasks_limit:
+            loop.run_until_complete(asyncio.sleep(5))
+        task = loop.create_task(archive_bvid(d, bvid))
+    
+    loop.run_until_complete(asyncio.sleep(5))
+    loop.close()
+
 
 def get_sess_data():
-    with open('sess_data.txt', 'r', encoding='utf-8') as f:
+    with open(os.path.expanduser('~/.sess_data.txt'), 'r', encoding='utf-8') as f:
         sess_data = f.read().strip()
     return sess_data
 
 
 if __name__ == '__main__':
-    main()
+    try:
+        main()
+    except KeyboardInterrupt:
+        print('KeyboardInterrupt')
+    finally:
+        # 显示终端光标
+        console = Console()
+        console.show_cursor()

tasks_limit.py

@@ -0,0 +1 @@
+tasks_limit = 2