最近再之乎上看到比较好的Python爬虫教程，看过之后对爬虫有了大概的了解，随后自己写了个爬取暴走漫画动图的爬虫练练手，另外附上Python爬虫教程的原始链接，完整看一遍教程之后还是会有很多收获的 ##源码 话不多说，直接上代码
1# -*- coding: UTF-8 -*- 2 3import requests 4import bs4 5import sys 6import os 7import re 8from multiprocessing.dummy import Pool as ThreadPool 9import urllib3 10from tqdm import tqdm 11import shutil 12 13baseUrl = 'http://baozoumanhua.com/catalogs/gif' 14 15curDir = os.getcwd() 16 17htmlDir = os.path.join(curDir, 'htmls') 18 19gifDir = os.path.join(curDir, 'gifs') 20 21gifMap = {} 22 23noneCnt = 0 24 25# win文件命名不允许使用的字符 26pat = re.compile(r'[\/|\?|\*|:|\||\\|<|>|\s|"]') 27 28total_pages = 1000 29 30get_gifs_bar = '' 31 32 33def get_html_text(url): 34 try: 35 r = requests.get(url, timeout=30) 36 r.raise_for_status() 37 r.encoding = r.apparent_encoding 38 return r.text 39 except: 40 return "Something Wrong!" 41 42 43def get_pages(num): 44 global get_pages_bar 45 get_pages_bar = tqdm(total=total_pages, ascii=True) 46 tqdm.write('.') 47 tqdm.write('.') 48 tqdm.write('.') 49 tqdm.write('Downloading web pages...') 50 num += 1 51 pool = ThreadPool(8) 52 pool.map(download_page, range(1, num)) 53 pool.close() 54 pool.join() 55 get_pages_bar.close() 56 57 58def get_gif_name(num, item): 59 global noneCnt 60 author = item.find('a', 'text-overflow').string + '：' 61 a_s = item.find_all('a') 62 for a in a_s: 63 if a.has_attr('data-full-url'): 64 gif_name = author + a.string 65 return gif_name 66 gif_name = author + 'NA' + str(noneCnt) 67 noneCnt += 1 68 return gif_name 69 70 71def get_gif_links(item): 72 imgs = item.find_all('img', 'lazy lazy-img none') 73 links = [] 74 for img in imgs: 75 if img.has_attr('data-original'): 76 links.append(img['data-original']) 77 return links 78 79 80def add_gifMap(name, links): 81 global gifMap 82 if len(links) < 1: 83 return 84 if len(links) == 1: 85 gif_name = name + '.gif' 86 gifMap[gif_name] = links[0] 87 return 88 for i in range(1, len(links) + 1): 89 gif_name = name + str(i) + '.gif' 90 gifMap[gif_name] = links[i - 1] 91 92 93def get_gifs(num): 94 tqdm.write('.') 95 tqdm.write('.') 96 tqdm.write('.') 97 tqdm.write('Parsing pages...') 98 num += 1 99 get_links_bar = tqdm(total=total_pages, ascii=True) 100 for n in range(1, num): 101 file_name = os.path.join(htmlDir, 'page' + str(n) + '.html') 102 soup = bs4.BeautifulSoup(open(file_name, 'rb'), 'lxml') 103 article = soup.find_all('div', 'article') 104 for item in article: 105 gif_name = get_gif_name(n, item) 106 gif_links = get_gif_links(item) 107 add_gifMap(gif_name, gif_links) 108 get_links_bar.update(1) 109 get_links_bar.close() 110 111 112def download_gif(name): 113 global gifMap 114 global pat 115 global get_gifs_bar 116 file_name = re.sub(pat, '_', name) 117 try: 118 if os.path.exists(os.path.join(htmlDir, 'gifs', file_name)): 119 return 120 r = requests.get(gifMap[name], timeout=30, verify=False) 121 r.raise_for_status() 122 with open(os.path.join(gifDir, file_name), 'wb') as fo: 123 fo.write(r.content) 124 except: 125 tqdm.write('Download ' + name + ' fail...') 126 finally: 127 get_gifs_bar.update(1) 128 129 130def downloader(): 131 total_gifs = len(gifMap.keys()) 132 tqdm.write('.') 133 tqdm.write('.') 134 tqdm.write('.') 135 tqdm.write('Downloading gifs...') 136 global get_gifs_bar 137 get_gifs_bar = tqdm(total=total_gifs, ascii=True) 138 pool = ThreadPool(8) 139 pool.map(download_gif, gifMap.keys()) 140 pool.close() 141 pool.join() 142 get_gifs_bar.close() 143 144 145def download_page(num): 146 url = baseUrl + '?page=' + str(num) 147 file_name = os.path.join(htmlDir, 'page' + str(num) + '.html') 148 with open(file_name, 'wb') as fo: 149 fo.write(get_html_text(url)) 150 get_pages_bar.update(1) 151 152 153def set_env(): 154 global total_pages 155 if os.path.exists(gifDir) and sum([len(x) for _, _, x in os.walk(gifDir)]) > 5000: 156 total_pages = 10 157 tqdm.write('Find many gifs in dir, just update gifs...') 158 if not os.path.exists(gifDir): 159 os.mkdir(gifDir) 160 if os.path.exists(htmlDir): 161 shutil.rmtree(htmlDir) 162 os.mkdir(htmlDir) 163 164 165def main(): 166 set_env() 167 get_pages(total_pages) 168 get_gifs(total_pages) 169 downloader() 170 shutil.rmtree(htmlDir) 171 tqdm.write('Congratulatins!!!') 172 tqdm.write('All pictures in folder : gifs...') 173 tqdm.write('Just open the folder and enjoy yourself!!!') 174 os.system('pause') 175 return 0 176 177 178if __name__ == "__main__": 179 urllib3.disable_warnings() 180 reload(sys) 181 sys.setdefaultencoding('utf-8') 182 sys.exit(main()) 183
程序运行示例 
首次执行会下载1000页的动图，请耐心等待，根据网络状况不同，可能需要30分钟的时间，并且确保磁盘有13G的空间 完整执行过一次该软件后，再次执行只会更新最新10页动图 执行完毕后，所有动图保存在当前文件夹下的gifs文件夹中