Python 使用selenium抓取网页文本和下载音频

Python 使用selenium抓取网页文本和下载音频

1#！\usr\bin\env python
2# -*- coding: utf-8 -*-
3
4'一个自动从https://podcast.duolingo.com/spanish中下载音频并且爬取文本的程序'
5'需要配置下载以下所需库，并且配置好webdriver.Chrome()，否则报错'
6
7from selenium import webdriver
8import requests
9import re
10import os
11import shelve
12                   
13def mainProc():
14    '主进程'
15    db = openDb()
16    get_pages(db)
17    get_episodes(db)
18    db.close() 
19
20def openDb():
21    '打开data文件，如果当前路径不存在，则新建文件并初始化'
22    filename = "data.dat"
23    if not os.path.exists(filename):
24        db = shelve.open("data", writeback=True)
25        db["pages"] = []
26        db["episodes"] = []
27    else:
28        db = shelve.open("data", writeback=True)
29
30    return db
31
32def get_pages(db):
33    '遍历获取所有页面的网址并保存到shelve文件中'
34    # 主页面
35    main = 'https://podcast.duolingo.com/spanish'  
36
37    # 循环遍历获取所有页面的网址
38    # 第一页则为主页面，不需要在main末尾添加i
39    #'https://podcast.duolingo.com/spanish2' 以此类推"
40    # 如果页面没有在文件中存在，则尝试访问页面，如果200成功，写入文本
41    
42    for i in range(1, 100):   
43        page = main if i == 1 else main + str(i)
44        if not page in db["pages"]:                                 
45            r = requests.get(page)
46            print(f'{page} with status code {r.status_code}.') 
47            if r.status_code != 200:                                
48                break 
49            db["pages"].append(page)
50            # 获取页面所有节目链接并补全连接
51            episodes = re.findall('entry-title">\s*<a href="(.*)" rel', r.text)
52            for episode in episodes:
53                episode = str(main[:-7]) + str(episode[2:])
54                db["episodes"].append(episode)
55                        
56def get_episodes(db):
57    '在每一页中遍历所有的单集网址'
58    for episode in db["episodes"]:
59        r = requests.get(episode)
60        print(f'{episode} with status code {r.status_code}.') 
61        if r.status_code != 200:
62            continue
63        # 将页面的文本写入文件中并下载音频
64        get_transcript(episode)
65        get_audios(r, episode)
66
67def get_transcript(episode):
68    # 获取节目单集网址中的文本
69    filename = 'transcript/' + episode.split('/')[-1] + '.txt'
70    if os.path.exists(filename):
71        print(filename, 'existed!')
72    else:
73        req = requests.get(episode)
74        print('{episode} with status code {status}.'.format(episode=episode, status=req.status_code))
75        if not os.path.exists('transcript'):
76            os.mkdir('transcript')
77        with open(filename, 'w+', encoding="utf-8") as fp:
78            for lines in re.findall('strong>(.*)</strong>(.*)</p>', req.text):
79                for line in lines:
80                    fp.write(line)
81                fp.write('\n\n')
82            print(filename, 'added!')
83
84def get_audios(r, episode):
85    audio = "https:" + re.findall('<iframe .* src="(.*)" height', r.text)[0]
86    # 自定义下载配置
87    chromeOptions = webdriver.ChromeOptions()
88    chromeOptions.add_argument("--ignore-certificate-errors")
89    prefs = {"download.default_directory":r"E:\Python\code\project\duolingo\audio"}
90    chromeOptions.add_experimental_option("prefs", prefs)
91    # 下载文件
92    print(audio)
93    browser = webdriver.Chrome(chrome_options=chromeOptions)
94    browser.get(audio)
95    if not os.path.exists("audio"):
96        os.mkdir("audio")
97    browser.find_element_by_id('download-player').click()
98    download_status = False
99    while not download_status:
100        download_status = True
101        for i in os.listdir('audio'):
102            if i.endswith(".crdownload"):
103                download_status = False
104                time.sleep(5)
105    browser.close()
106
107if __name__ == "__main__":
108    mainProc()