1.豆瓣采集
1 1 #coding:utf-8 2 2 #采集豆瓣书信息和图片，写进数据库 3 3 4 4 from urllib import request 5 5 # from bs4 import BeautifulSoup 6 6 from lxml import etree 7 7 import json,pymysql 8 8 9 9 # from my_pymysql import pymysql 1010 1111 url="https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4" 1212 headers={ 1313 'Host':'book.douban.com', 1414 'Upgrade-Insecure-Requests':'1', 1515 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36' 1616 } 1717 req = request.Request(url=url,headers=headers,method="GET") 1818 content = request.urlopen(req).read().decode("utf-8") 1919 content_dict=etree.HTML(content) #格式化 2020 # print(content_dict) 2121 content_dict_allli = content_dict.xpath(r'//*[@id="subject_list"]/ul/li') #拿到列表 2222 info_all = '' 2323 2424 for li in content_dict_allli: 2525 # 书名/标题 2626 title_list = li.xpath(r'div[2]/h2/a/@title') #取标签里的内容，注意地址是相对地址，不能直接拿来用 （注：和bs4不一样） 2727 title =title_list[0] 2828 title=title.replace(" ",'') 2929 print(title) 3030 #信息 作者、出版社 3131 info_list = li.xpath(r'div[2]/div[1]/text()') 3232 author = info_list[0].split('/')[0] 3333 author = author.replace('\n','').replace(" ",'') 3434 chubanshe = info_list[0].split('/')[1] 3535 print(author) 3636 print(chubanshe) 3737 #评分 3838 pingfen_list = li.xpath(r'div[2]/div[2]/span[2]/text()') 3939 pingfen = pingfen_list[0] 4040 print(pingfen) 4141 4242 #图片 4343 img_net_addr =li.xpath(r'div[1]/a/img/@src') 4444 img_net_addr = img_net_addr[0] 4545 print(img_net_addr) 4646 data = request.urlopen(img_net_addr).read() 4747 img_name =str('douban/') + title + str('.jpg') 4848 with open(img_name,'wb')as f: 4949 f.write(data) 5050 5151 #数据库 5252 db = pymysql.connect(host='localhost',port=3306,user="root",password='root',db='douban',charset='utf8') # 5353 cur=db.cursor() 5454 sql = "insert into douban(title,author,chubanshe,pingfen)values('%s','%s','%s','%s')"%(title,author,chubanshe,pingfen) 5555 cur.execute(sql) 5656 db.commit() 5757 5858 db.close()
采集豆瓣书信息和图片；带请求头、存数据库、图片；写进数据库
2.链家
1#coding:utf-8 2#完成，，取出链家数据存到文件里 3from urllib import request,error 4from bs4 import BeautifulSoup 5import pymysql 6 7# from my_pymysql import pymysql #引入数据库 8#创建数据库 9db = pymysql.connect(host='localhost',user='root',password='root',db='lianjia',charset='utf8') 10cur = db.cursor() #实例化游标 11 12for i in range(1,33): 13 req=request.urlopen('https://xa.lianjia.com/ershoufang/pg'+str(i)).read().decode('utf-8') 14 req_bs4 = BeautifulSoup(req,'html.parser') #建立对象，才能用bs4 15 body_ul=req_bs4.find('ul',class_="sellListContent") 16 try: 17 s='' 18 for li in body_ul: 19 # info_all = li.find('div',class_="info clear").get_text() #全部信息 20 tit = li.find('div',class_="title").get_text() #标题 21 addr = li.find('div',class_="houseInfo").get_text() #地址 22 pric = li.find('div',class_="totalPrice").get_text() #价格 23 s+=tit 24 s+=addr 25 s+=pric 26 s+='\n\n' 27 print(i) #提示采集的位置 28 # 采集图片开始++++++++++++++++++++++++++++++++++++++++++++ 29 img = li.find("img", class_='lj-lazy')['data-original'] #图片地址 30 img_format = img.split('.')[-1] # 用点隔开，取图片的后缀 31 img_name = 'lianjia/images/' + li.find("img", class_='lj-lazy')['alt'] + '.' + img_format # 名字 32 adr = request.urlopen(img).read() # 读取图片地址，拿到字节流形式的图片，，写进去 33 try: #；空的话就跳过 34 with open(img_name, 'wb')as f: 35 f.write(adr) 36 except: 37 pass 38 # 采集图片完毕---------------------------- 39 #存到数据库 40 sql = "insert into lianjia_hotel(title,address) values ('%s','%s')"%(tit,addr) 41 cur.execute(sql) 42 db.commit() 43 except: 44 print("本页完毕~") 45#最后再关闭数据库 46db.close() 47 48#写到一个txt文件里面 49# with open('lianjia/lianjia.txt','w',encoding="utf-8")as f: 50# f.write(s)
链家下载，文字与图片，用bs4解析
3.今日头条
1from selenium import webdriver 2from lxml import etree 3from pyquery import PyQuery as pq 4import time 5 6driver = webdriver.Chrome() 7driver.maximize_window() 8driver.get('https://www.toutiao.com/') 9driver.implicitly_wait(10) 10driver.find_element_by_link_text('科技').click() 11driver.implicitly_wait(10) 12for x in range(3): 13 js="var q=document.documentElement.scrollTop="+str(x*500) 14 driver.execute_script(js) 15 time.sleep(2) 16 17time.sleep(5) 18page = driver.page_source 19doc = pq(page) 20doc = etree.HTML(str(doc)) 21contents = doc.xpath('//div[@class="wcommonFeed"]/ul/li') 22print(contents) 23for x in contents: 24 title = x.xpath('div/div[1]/div/div[1]/a/text()') 25 if title: 26 title = title[0] 27 with open('toutiao.txt','a+',encoding='utf8')as f: 28 f.write(title+'\n') 29 print(title) 30 else: 31 pass
今日头条，selenium控制翻页
4.微信群信息(包括成员)和联系人
1# -*- coding:utf-8 -*- 2''' 3扫码登陆微信后获取该微信账号的微信群(包括群内人员)和通讯录联系人信息【注：好像不全】 4''' 5 6import os 7import re 8import time 9import sys 10import subprocess 11import requests 12import xml.dom.minidom 13import json 14 15 16# 微信登陆 17class WebwxLogin(object): 18 def __init__(self): 19 self.session = requests.session() 20 self.headers = { 21 'User-Agent': 'Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0'} 22 self.QRImgPath = os.path.split(os.path.realpath(__file__))[0] + os.sep + 'webWeixinQr.jpg' 23 self.uuid = '' 24 self.tip = 0 25 self.base_uri = '' 26 self.redirect_uri = '' 27 self.skey = '' 28 self.wxsid = '' 29 self.wxuin = '' 30 self.pass_ticket = '' 31 self.deviceId = 'e000000000000000' 32 self.BaseRequest = {} 33 self.ContactList = [] 34 self.My = [] 35 self.SyncKey = '' 36 37 def getUUID(self): 38 39 url = 'https://login.weixin.qq.com/jslogin' 40 params = { 41 'appid': 'wx782c26e4c19acffb', 42 'redirect_uri': 'https://wx.qq.com/cgi-bin/mmwebwx-bin/webwxnewloginpage', 43 'fun': 'new', 44 'lang': 'zh_CN', 45 '_': int(time.time() * 1000), # 时间戳 46 } 47 48 response = self.session.get(url, params=params) 49 target = response.content.decode('utf-8') 50 51 pattern = r'window.QRLogin.code = (\d+); window.QRLogin.uuid = "(\S+?)"' 52 ob = re.search(pattern, target) # 正则提取uuid 53 54 code = ob.group(1) 55 self.uuid = ob.group(2) 56 57 if code == '200': # 判断请求是否成功 58 return True 59 60 return False 61 62 def showQRImage(self): 63 64 url = 'https://login.weixin.qq.com/qrcode/' + self.uuid 65 response = self.session.get(url) 66 67 self.tip = 1 68 69 with open(self.QRImgPath, 'wb') as f: 70 f.write(response.content) 71 f.close() 72 # 打开二维码 73 if sys.platform.find('darwin') >= 0: 74 subprocess.call(['open', self.QRImgPath]) # 苹果系统 75 elif sys.platform.find('linux') >= 0: 76 subprocess.call(['xdg-open', self.QRImgPath]) # linux系统 77 else: 78 os.startfile(self.QRImgPath) # windows系统 79 80 print('请使用微信扫描二维码登录') 81 82 def checkLogin(self): 83 84 url = 'https://login.weixin.qq.com/cgi-bin/mmwebwx-bin/login?tip=%s&uuid=%s&_=%s' % ( 85 self.tip, self.uuid, int(time.time() * 1000)) 86 87 response = self.session.get(url) 88 target = response.content.decode('utf-8') 89 90 pattern = r'window.code=(\d+);' 91 ob = re.search(pattern, target) 92 code = ob.group(1) 93 94 if code == '201': # 已扫描 95 print('成功扫描,请在手机上点击确认登录') 96 self.tip = 0 97 elif code == '200': # 已登录 98 print('正在登录中...') 99 regx = r'window.redirect_uri="(\S+?)";' 100 ob = re.search(regx, target) 101 self.redirect_uri = ob.group(1) + '&fun=new' 102 self.base_uri = self.redirect_uri[:self.redirect_uri.rfind('/')] 103 elif code == '408': # 超时 104 pass 105 106 return code 107 108 def login(self): 109 110 response = self.session.get(self.redirect_uri, verify=False) 111 data = response.content.decode('utf-8') 112 113 doc = xml.dom.minidom.parseString(data) 114 root = doc.documentElement 115 # 提取响应中的参数 116 for node in root.childNodes: 117 if node.nodeName == 'skey': 118 self.skey = node.childNodes[0].data 119 elif node.nodeName == 'wxsid': 120 self.wxsid = node.childNodes[0].data 121 elif node.nodeName == 'wxuin': 122 self.wxuin = node.childNodes[0].data 123 elif node.nodeName == 'pass_ticket': 124 self.pass_ticket = node.childNodes[0].data 125 126 if not all((self.skey, self.wxsid, self.wxuin, self.pass_ticket)): 127 return False 128 129 self.BaseRequest = { 130 'Uin': int(self.wxuin), 131 'Sid': self.wxsid, 132 'Skey': self.skey, 133 'DeviceID': self.deviceId, 134 } 135 136 return True 137 138 def webwxinit(self): 139 140 url = self.base_uri + \ 141 '/webwxinit?pass_ticket=%s&skey=%s&r=%s' % ( 142 self.pass_ticket, self.skey, int(time.time() * 1000)) 143 params = { 144 'BaseRequest': self.BaseRequest 145 } 146 147 h = self.headers 148 h['ContentType'] = 'application/json; charset=UTF-8' 149 response = self.session.post(url, data=json.dumps(params), headers=h, verify=False) 150 data = response.content.decode('utf-8') 151 print(data) 152 153 dic = json.loads(data) 154 self.ContactList = dic['ContactList'] 155 self.My = dic['User'] 156 157 SyncKeyList = [] 158 for item in dic['SyncKey']['List']: 159 SyncKeyList.append('%s_%s' % (item['Key'], item['Val'])) 160 self.SyncKey = '|'.join(SyncKeyList) 161 162 ErrMsg = dic['BaseResponse']['ErrMsg'] 163 164 Ret = dic['BaseResponse']['Ret'] 165 if Ret != 0: 166 return False 167 168 return True 169 170 def webwxgetcontact(self): 171 172 url = self.base_uri + \ 173 '/webwxgetcontact?pass_ticket=%s&skey=%s&r=%s' % ( 174 self.pass_ticket, self.skey, int(time.time())) 175 176 h = self.headers 177 h['ContentType'] = 'application/json; charset=UTF-8' 178 response = self.session.get(url, headers=h, verify=False) 179 data = response.content.decode('utf-8') 180 # print(data) 181 182 dic = json.loads(data) 183 MemberList = dic['MemberList'] 184 185 # 倒序遍历,不然删除的时候出问题.. 186 SpecialUsers = ["newsapp", "fmessage", "filehelper", "weibo", "qqmail", "tmessage", "qmessage", "qqsync", 187 "floatbottle", "lbsapp", "shakeapp", "medianote", "qqfriend", "readerapp", "blogapp", 188 "facebookapp", "masssendapp", 189 "meishiapp", "feedsapp", "voip", "blogappweixin", "weixin", "brandsessionholder", 190 "weixinreminder", "wxid_novlwrv3lqwv11", "gh_22b87fa7cb3c", "officialaccounts", 191 "notification_messages", "wxitil", "userexperience_alarm"] 192 for i in range(len(MemberList) - 1, -1, -1): 193 Member = MemberList[i] 194 if Member['VerifyFlag'] & 8 != 0: # 公众号/服务号 195 MemberList.remove(Member) 196 elif Member['UserName'] in SpecialUsers: # 特殊账号 197 MemberList.remove(Member) 198 elif Member['UserName'].find('@@') != -1: # 群聊 199 MemberList.remove(Member) 200 elif Member['UserName'] == self.My['UserName']: # 自己 201 MemberList.remove(Member) 202 203 return MemberList 204 205 def main(self): 206 if not self.getUUID(): 207 print('获取uuid失败') 208 return 209 210 self.showQRImage() 211 time.sleep(1) 212 213 while self.checkLogin() != '200': 214 pass 215 216 os.remove(self.QRImgPath) 217 218 if not self.login(): 219 print('登录失败') 220 return 221 # 登录完成, 下面查询好友 222 if not self.webwxinit(): 223 print('初始化失败') 224 return 225 226 MemberList = self.webwxgetcontact() 227 228 print('通讯录共%s位好友' % len(MemberList)) 229 230 for x in MemberList: 231 sex = '未知' if x['Sex'] == 0 else '男' if x['Sex'] == 1 else '女' 232 print('昵称:%s, 性别:%s, 备注:%s, 签名:%s' % (x['NickName'], sex, x['RemarkName'], x['Signature'])) 233 234if __name__ == '__main__': 235 print('开始') 236 wx = WebwxLogin() 237 wx.main()
爬取微信群信息(包括成员)和联系人信息
5.爬取淘宝固定类别商品信息+保存到mysql数据库【格式很规范】
1import requests 2import re 3import pymysql 4 5 6def getHTMLtext(url): 7 try: 8 r=requests.get(url,timeout=100) 9 r.raise_for_status() 10 r.encoding=r.apparent_encoding 11 return r.text 12 except: 13 return "" 14def getpage(itl,html): 15 try: 16 plt=re.findall(r'"view_price":"[\d.]*"',html) 17 nlt=re.findall(r'"raw_title":".*?"',html) 18 for i in range(len(plt)): 19 price = eval(plt[i].split(':')[1]) # eval(fun,obj) 20 title = eval(nlt[i].split(':')[1]) 21 itl.append([price, title]) 22 except: 23 print("") 24 25 26def printgoods(itl): 27 tplt = "{:2}\t{:8}\t{:16}" 28 print(tplt.format("序号", "价格", "商品名称")) 29 30 count = 0 31 conn = pymysql.connect(host='127.0.0.1', user='root', password='123456', db='company',charset="utf8") 32 33 cur = conn.cursor() 34 35 sqlc = ''' 36 create table coffee( 37 id int(11) not null auto_increment primary key, 38 name varchar(255) not null, 39 price float not null)DEFAULT CHARSET=utf8; 40 ''' 41 42 try: 43 A = cur.execute(sqlc) 44 conn.commit() 45 print('成功') 46 except: 47 print("错误") 48 for g in itl: 49 count = count + 1 50 b=tplt.format(count, g[0], g[1]) 51 52 53 54 sqla = ''' 55 insert into coffee(name,price) 56 values(%s,%s); 57 ''' 58 try: 59 B = cur.execute(sqla,(g[1],g[0])) 60 conn.commit() 61 print('成功') 62 except: 63 print("错误") 64 65 # save_path = 'D:/taobao.txt' 66 # f=open(save_path,'a') 67 # 68 # f.write(b+'\n') 69 # f.close() 70 71 conn.commit() 72 cur.close() 73 conn.close() 74 75 76def main(): 77 goods="咖啡" 78 depth =2 79 start_url='https://s.taobao.com/search?q='+goods 80 List =[] 81 for i in range(depth): 82 try: 83 url =start_url +"&s="+ str(i*44) 84 html=getHTMLtext(url) 85 getpage(List,html) 86 except: 87 continue 88 89 90 print(printgoods(List)) 91 # savefiles(data) 92 93 94 95 96main()
淘宝信息采集+保存到Mysql数据库