集成爬虫

Bilibili.py

@@ -2,5 +2,5 @@
 from utils.Clip_bili import *
 
 if __name__ == '__main__':
-    # downloads()
-    downloadsAudio()
+    downloads()
+    # downloadsAudio()

bili_jicheng.py

@@ -0,0 +1,90 @@
+# 导入数据请求模块
+import requests
+# 导入正则表达式模块
+import re
+# 导入json模块
+import json
+# 导入合并模块
+from moviepy.editor import *
+
+# 设置
+url = 'https://www.bilibili.com/video/BV1LxSuYDEBR/?spm_id_from=333.1007.tianma.1-1-1.click&vd_source=f6247aa12dae1ff1bce74ef0af381757'
+headers = {
+        "Referer": url,
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        'cookie': 'buvid3=D82F9D33-E75C-8D6E-6D2D-D128A352B3EC99573infoc; '
+                  'b_nut=1730271799; '
+                  '_uuid=363DA659-CAA7-AEE3-31EF-105ADAD95EFA299807infoc; '
+                  'enable_web_push=DISABLE; '
+                  'buvid4=A8A4CC10-7B3C-8459-B07C-EB4E11E948D401840-024103007-fmI38SPP%2F5%2BxXWrCTYgty62zOzJos1uEW8lnI07A3XTkLJ6dLJvsmsAJBqLNPrF9; '
+                  'rpdid=0zbfvUnKtc|3XngooMd|T30|3w1T62JV; '
+                  'header_theme_version=CLOSE; '
+                  'DedeUserID=35990046; '
+                  'DedeUserID__ckMd5=bf34a8eab39f0c11; '
+                  'buvid_fp_plain=undefined; '
+                  'LIVE_BUVID=AUTO3317306871682983; '
+                  'CURRENT_QUALITY=80; '
+                  'fingerprint=afd86aa8ba10b52e21767b02b25b933b; '
+                  'buvid_fp=afd86aa8ba10b52e21767b02b25b933b; '
+                  'CURRENT_FNVAL=4048; '
+                  'bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MzIyNjg3OTMsImlhdCI6MTczMjAwOTUzMywicGx0IjotMX0.XrtK4fW5yyBnEWTn4gQeSTZAh_8KweejUzSk9tPDC08; '
+                  'bili_ticket_expires=1732268733; '
+                  'PVID=3; bmg_af_switch=1; '
+                  'bmg_src_def_domain=i2.hdslb.com; '
+                  'b_lsid=AEA3F28F_1934D6025DD; '
+                  'SESSDATA=994c56c3%2C1747721973%2C32480%2Ab1CjC6H2WiSR2TdOw7R4ED2MVj0RDd_OeOczPS9dMUYYuj8I9ilx01iewdYO_1V7J5VSoSVkVtaE1tV0hxRHIwb2tndmJraTd3X3pvcXBleHI4NDhRN3o5YnAtOWlVX1lwVXhTc210RS05VDFkUnVaMUQ3MjBiQ0h1dENpdi1VOU9hLUxycHUzR2RBIIEC; '
+                  'bili_jct=4cd051b6642a32eeb57f558eac9f02f7; '
+                  'sid=6w3kthlu; '
+                  'home_feed_column=5; '
+                  'browser_resolution=1528-712; '
+                  'bsource=search_bing; '
+                  'bp_t_offset_35990046=1002173800399241216'
+}
+
+# 发送请求
+response = requests.get(url=url, headers=headers)
+html = response.text
+
+# 提取,合法化并显示视频标题
+title = re.findall('title="(.*?)"', html)[0]
+if title:
+    illegal_chars = fr'<|>\/:"*?'
+    def remove_illegal_chars(title_ill):
+        for char in illegal_chars:
+            title_ill = title_ill.replace(char, "")
+        return title_ill
+    title = remove_illegal_chars(title)
+print(title)
+
+# 提取视频信息
+info = re.findall('window.__playinfo__=(.*?)</script>', html)[0]
+json_data = json.loads(info)
+
+# 提取音视频链接
+# TODO:设置数值来声明文件品质
+video_url = json_data['data']['dash']['video'][1]['baseUrl']
+audio_url = json_data['data']['dash']['audio'][0]['baseUrl']
+
+# 获取音视频内容
+video_content = requests.get(url=video_url, headers=headers).content
+audio_content = requests.get(url=audio_url, headers=headers).content
+
+# 设置保存地址
+A_path = 'D:/pyp/Spider/bilibili_pachong/bili_test/audio/'
+V_path = 'D:/pyp/Spider/bilibili_pachong/bili_test/video/'
+Synth_path = 'D:/pyp/Spider/bilibili_pachong/bili_test/File/'
+
+# 保存文件数据
+with open(V_path + title + '.mp4', mode='wb') as v:
+    v.write(video_content)
+with open(A_path + title + '.mp3', mode='wb') as a:
+    a.write(audio_content)
+
+# 合并视频
+audio_path = fr"{A_path}{title}{'.mp3'}"
+audio_clip = AudioFileClip(audio_path)
+video_path = fr"{V_path}{title}{'.mp4'}"
+video_clip = VideoFileClip(video_path)
+audio_clip = audio_clip.set_duration(video_clip.duration)
+video_clip_with_audio = video_clip.set_audio(audio_clip)
+video_clip_with_audio.write_videofile(fr"{Synth_path}{title}{'.mp4'}", codec="libx264", audio_codec="aac")

utils/GetResponse.py

@@ -8,31 +8,15 @@ import requests
 
 # //视频音频请求头//
 headers_bili_av = {
-    'Accept': '*/*',
-    'Accept-Language': 'zh,en-US;q=0.7,en;q=0.3',
-    'Accept-Encoding': 'gzip, deflate, br',
-    'TE': 'Trailers',
-    'Range': 'bytes=0-',
-    'Origin': 'https://www.bilibili.com',
-    'Connection': 'keep-alive',
-    'DNT': '1',
-    'referer': 'https://www.bilibili.com',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0'
+        "Referer": url,
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        'cookie': 'buvid3=D82F9D33-E75C-8D6E-6D2D-D128A352B3EC99573infoc; b_nut=1730271799; _uuid=363DA659-CAA7-AEE3-31EF-105ADAD95EFA299807infoc; enable_web_push=DISABLE; buvid4=A8A4CC10-7B3C-8459-B07C-EB4E11E948D401840-024103007-fmI38SPP%2F5%2BxXWrCTYgty62zOzJos1uEW8lnI07A3XTkLJ6dLJvsmsAJBqLNPrF9; rpdid=0zbfvUnKtc|3XngooMd|T30|3w1T62JV; header_theme_version=CLOSE; DedeUserID=35990046; DedeUserID__ckMd5=bf34a8eab39f0c11; buvid_fp_plain=undefined; LIVE_BUVID=AUTO3317306871682983; CURRENT_QUALITY=80; fingerprint=afd86aa8ba10b52e21767b02b25b933b; buvid_fp=afd86aa8ba10b52e21767b02b25b933b; CURRENT_FNVAL=4048; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MzIyNjg3OTMsImlhdCI6MTczMjAwOTUzMywicGx0IjotMX0.XrtK4fW5yyBnEWTn4gQeSTZAh_8KweejUzSk9tPDC08; bili_ticket_expires=1732268733; PVID=3; bmg_af_switch=1; bmg_src_def_domain=i2.hdslb.com; b_lsid=AEA3F28F_1934D6025DD; SESSDATA=994c56c3%2C1747721973%2C32480%2Ab1CjC6H2WiSR2TdOw7R4ED2MVj0RDd_OeOczPS9dMUYYuj8I9ilx01iewdYO_1V7J5VSoSVkVtaE1tV0hxRHIwb2tndmJraTd3X3pvcXBleHI4NDhRN3o5YnAtOWlVX1lwVXhTc210RS05VDFkUnVaMUQ3MjBiQ0h1dENpdi1VOU9hLUxycHUzR2RBIIEC; bili_jct=4cd051b6642a32eeb57f558eac9f02f7; sid=6w3kthlu; home_feed_column=5; browser_resolution=1528-712; bsource=search_bing; bp_t_offset_35990046=1002173800399241216'
 }
 # //准许请求头//
 headers_bili_ref = {
-    'Host': 'cn-gdgz4-cmcc-v-10.bilivideo.com',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0',
-    'Accept': '*/*',
-    'Accept-Language': 'zh,en-US;q=0.7,en;q=0.3',
-    'Accept-Encoding': 'gzip, deflate, br',
-    'Access-Control-Request-Method': 'GET',
-    'Access-Control-Request-Headers': 'range',
-    'Referer': url,
-    'Origin': base_url,
-    'DNT': '1',
-    'Connection': 'keep-alive',
-    'Cache-Control': 'max-age=0'
+        "Referer": url,
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        'cookie': 'buvid3=D82F9D33-E75C-8D6E-6D2D-D128A352B3EC99573infoc; b_nut=1730271799; _uuid=363DA659-CAA7-AEE3-31EF-105ADAD95EFA299807infoc; enable_web_push=DISABLE; buvid4=A8A4CC10-7B3C-8459-B07C-EB4E11E948D401840-024103007-fmI38SPP%2F5%2BxXWrCTYgty62zOzJos1uEW8lnI07A3XTkLJ6dLJvsmsAJBqLNPrF9; rpdid=0zbfvUnKtc|3XngooMd|T30|3w1T62JV; header_theme_version=CLOSE; DedeUserID=35990046; DedeUserID__ckMd5=bf34a8eab39f0c11; buvid_fp_plain=undefined; LIVE_BUVID=AUTO3317306871682983; CURRENT_QUALITY=80; fingerprint=afd86aa8ba10b52e21767b02b25b933b; buvid_fp=afd86aa8ba10b52e21767b02b25b933b; CURRENT_FNVAL=4048; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MzIyNjg3OTMsImlhdCI6MTczMjAwOTUzMywicGx0IjotMX0.XrtK4fW5yyBnEWTn4gQeSTZAh_8KweejUzSk9tPDC08; bili_ticket_expires=1732268733; PVID=3; bmg_af_switch=1; bmg_src_def_domain=i2.hdslb.com; b_lsid=AEA3F28F_1934D6025DD; SESSDATA=994c56c3%2C1747721973%2C32480%2Ab1CjC6H2WiSR2TdOw7R4ED2MVj0RDd_OeOczPS9dMUYYuj8I9ilx01iewdYO_1V7J5VSoSVkVtaE1tV0hxRHIwb2tndmJraTd3X3pvcXBleHI4NDhRN3o5YnAtOWlVX1lwVXhTc210RS05VDFkUnVaMUQ3MjBiQ0h1dENpdi1VOU9hLUxycHUzR2RBIIEC; bili_jct=4cd051b6642a32eeb57f558eac9f02f7; sid=6w3kthlu; home_feed_column=5; browser_resolution=1528-712; bsource=search_bing; bp_t_offset_35990046=1002173800399241216'
 }
 
 headers = {
@@ -48,7 +32,7 @@ video_json = {
 }
 
 def GetResponse_AV():
-    response_audio = requests.get(url = url,headers = headers_bili_av)
+    response_audio = requests.get(url = url,headers = headers_bili_av,stream = True)
     return response_audio
 
 # 检查

utils/GetVideoInfo.py

@@ -27,12 +27,14 @@ def GetAudioInfo() -> str:
     audio_url = json_data_audio['data']['dash']['audio'][0]['baseUrl']
     return audio_url
 
-def GetTitile():
-    response_audio = GetResponse_AV()
-    html = response_audio.text
-    title = re.findall('<title data-vue-meta="true">(.*?)</title>', html)[0]
-    print("原名为：",title)
-    video_json[title] = title
+def GetTitile() -> str:
+    # response_audio = GetResponse_AV()
+    # html = response_audio.text
+    # title = re.findall('<title data-vue-meta="true">(.*?)</title>', html)[0]
+    # print("原件名称:",title)
+    # video_json[title] = title
+    html = GetResponse_AV().text
+    title = re.findall('title="(.*?)"', html)[0]
 
     # if not title:
     #     title = '未知'
@@ -45,11 +47,10 @@ def GetTitile():
         title = remove_illegal_chars(title)
     else:
         return None
-    print('文件名为:',title)
     # elif len(title) > 30:
-    #     # 如果名字过长，就取前20对反爬虫策略有一定的反制手段，如使用代理IP、设置随机访问时间、获取ajax等个字符
     #     title = title[:30]
-    # return title
+    print('合法名称:',title)
+    return title
 
 # 仅用作测试
 def GetHTML():

utils/Save_bili.py

@@ -4,6 +4,7 @@
 # @File    : Save_bili.py
 
 from utils.GetResponse import *
+import requests
 
 def SaveAV(session: requests.session(),url,title,v_url,a_url):
     # 设置请求头
@@ -16,14 +17,11 @@ def SaveAV(session: requests.session(),url,title,v_url,a_url):
 
     A_content = session.get(a_url, headers=headers_bili_av).content
     V_content = session.get(v_url, headers=headers_bili_av).content
-    print('开始下载音频...')
+
     with open(Audio_path + title + Audio_format, mode='wb') as audio:
         audio.write(A_content)
-    print('音频下载完成'
-          '开始下载视频...')
     with open(Video_path + title + Video_format, mode='wb') as video:
         video.write(V_content)
-    print('视频下载完成')
 
 def SaveAudio(session,url,title,a_url):
     # 设置请求头
@@ -35,7 +33,8 @@ def SaveAudio(session,url,title,a_url):
     A_content = session.get(a_url, headers=headers_bili_av).content
 
     # 下载程序
-    print(f'开始下载音频...路径为:',Audio_path)
+    print('开始下载音频...')
+    print('路径为:',Audio_path)
     with open(Audio_path + title + Audio_format, mode='wb') as audio:
         audio.write(A_content)
     print('音频下载结束')

utils/setting.py

@@ -6,9 +6,9 @@
 Audio_format = '.mp3'
 Video_format = '.mp4'
 
-Audio_path = r'D:/pyp/Spider/bilibili_pachong/bilibili_files/Audio/'
-Video_path = r'D:/pyp/Spider/bilibili_pachong/bilibili_files/Video/'
-Synth_path = r'D:/pyp/Spider/bilibili_pachong/bilibili_files/Files/'
+Audio_path = 'D:/pyp/Spider/bilibili_pachong/bilibili_files/Audio/'
+Video_path = 'D:/pyp/Spider/bilibili_pachong/bilibili_files/Video/'
+Synth_path = 'D:/pyp/Spider/bilibili_pachong/bilibili_files/Files/'
 
-url = 'https://www.bilibili.com/video/BV1hkUtYZEQf/?spm_id_from=333.1007.top_right_bar_window_default_collection.content.click&vd_source=f6247aa12dae1ff1bce74ef0af381757'
+url = 'https://www.bilibili.com/video/BV1LxSuYDEBR/?spm_id_from=333.1007.tianma.1-1-1.click&vd_source=f6247aa12dae1ff1bce74ef0af381757'
 base_url = 'https://www.bilibili.com'