使用Python抓取helloworld上的文章

嘿嘿嘿，py真好玩

代码写得很渣，本着能跑就好的原则，就这样啦~

1import os
2from requests_html import HTMLSession
3import sqlite3
4from pathlib import Path
5
6BASE_DIR = Path(__file__).resolve().parent.parent
7domain = "https://www.helloworld.net"
8
9class Helloworld:
10
11    def __init__(self):
12        self.con = sqlite3.connect(str(BASE_DIR) + "\\data\\helloworld.db", timeout=10)
13        # 引入sqlite3 是想存储到数据库来着
14        self.session = HTMLSession()
15
16    def getContent(self, p, saveToFile=False):
17        req = self.session.get(domain + p)
18        p = p.split('/')
19        if req.status_code != 200:
20            print("获取失败")
21            return False
22        author = req.html.find(".author-info", first=True)
23        data = {}
24        data['pubtime'] = author.xpath('//span[@title="发布时间"]//time/text()')[0].strip()
25        data['yuedushu'] = author.xpath('//span[@title="阅读数"]/text()')[0].strip()
26        data['zan'] = author.xpath('//span[@title="点赞数"]/text()')[0].strip()
27        data['shoucang'] = author.xpath('//span[@title="收藏数"]/text()')[0].strip()
28        data['content'] = req.html.find("article>div[id='htmlContent']", first=True).html
29        print(data)
30        if saveToFile:
31            data['save_path'] = os.path.join(BASE_DIR, "data") + p[2] + ".html"
32            # 创建文件
33            f = open(data['save_path'], 'wb')
34            f.write(bytes(data['content'], encoding="utf8"))
35            f.close()
36        return data
37
38    def getArticleList(self, hot=False, callback=None):
39        url = domain
40        if hot:
41            # 是否按照热门爬取
42            url += "?sort = hottest"
43
44        req = self.session.get(url)
45        if req.status_code != 200:
46            print("获取失败")
47            return False
48        blog_list_container = req.html.find(".blog_list_container>div.blog-item")
49        dataSet = []
50        for item in blog_list_container:
51            data = {}
52            author = {}
53            leftItem = item.find(".item-left", first=True)
54            data['p'] = leftItem.xpath('//a/@href')[0]
55            data['short_desc'] = leftItem.find("p", first=True).text
56            author['path'] = leftItem.find(".infos .pre-infos", first=True).xpath("//a/@href")[0]
57            author['portrait'] = leftItem.find(".infos .pre-infos a", first=True).xpath("//img/@src")[0]
58            author['nickname'] = leftItem.find(".infos .pre-infos .nickname", first=True).text
59            data['time'] = leftItem.find(".infos .pre-infos .time", first=True).text
60            data['yuedushu'] = \
61                leftItem.find(".infos .after-infos", first=True).xpath(
62                    '//div[@title="阅读数"]//span[@class="name"]/text()')[
63                    0].strip()
64            data['zan'] = \
65                leftItem.find(".infos .after-infos", first=True).xpath(
66                    '//div[@title="点赞数"]//span[@class="name"]/text()')[
67                    0].strip()
68            rightItem = item.find(".item-right", first=True)
69            data['cover_image'] = ''
70            if rightItem:
71                # 获取封面
72                data['cover_image'] = rightItem.xpath("//img/@src")
73                if data['cover_image']:
74                    data['cover_image'] = data['cover_image'][0]
75            data['author'] = author
76            # 获取文章内容部分
77            dataSet.append(data)
78            if callback:
79                callback(data)
80            print(data)
81        return dataSet
82
83
84def mycallback(data):
85    c = Helloworld().getContent(p=data.p, saveToFile=False)
86    # 嘿嘿嘿
87    
88if __name__ == '__main__':
89    Helloworld().getArticleList(callback=mycallback)
90    # c = Helloworld().getContent(p='/p/14G4HKmhx2FLe')

文章有个分页的问题，可以用selenium解决吧