目的：
使用requests库以及xpath解析进行实验楼所有课程，存入MySQL数据 库中。
准备工作：
首先安装，requests库，lxml库，以及peewee库。在命令行模式，使用以下命令。
1pip install requests 2pip install lxml 3pip install peewee
然后，就可以打开编辑器编写代码了。
代码:
1 1 # 引入所需要的库 2 2 import time 3 3 import requests 4 4 from peewee import * 5 5 from lxml import etree 6 6 # 这个程序要倒着看 7 7 8 8 # 这个是连接数据库的，名字和密码根据自己情况修改 9 9 db = MySQLDatabase('shiyanlou', user='root', passwd='xxxxxx') 10 10 11 11 12 12 class Course(Model): 13 13 title = CharField() 14 14 teacher = CharField() 15 15 teacher_courses = IntegerField() 16 16 tag = CharField() 17 17 study_num = IntegerField() 18 18 content = CharField() 19 19 20 20 class Meta: 21 21 database = db 22 22 23 23 24 24 Course.create_table() 25 25 26 26 27 27 def parse_content(url, title, tag, study_num): 28 28 print('课程地址：' + url) 29 29 res = requests.get(url) 30 30 xml = etree.HTML(res.text) 31 31 # 获取页面里的简介 32 32 try: 33 33 content = xml.xpath('//meta[@name="description"]/@content')[0] 34 34 except Exception as e: 35 35 content = '无' 36 36 # 获取老师名字 37 37 try: 38 38 teacher = xml.xpath( 39 39 '//div[@class="sidebox mooc-teacher"]//div[@class="mooc-info"]/div[@class="name"]/strong/text()')[0] 40 40 except Exception as e: 41 41 teacher = '匿名' 42 42 # 获取老师发表课程数目 43 43 try: 44 44 teacher_courses = xml.xpath( 45 45 '//div[@class="sidebox mooc-teacher"]//div[@class="mooc-info"]/div[@class="courses"]/strong/text()')[0] 46 46 except Exception as e: 47 47 teacher_courses = '未知' 48 48 # 存入数据库 49 49 try: 50 50 course = Course(title=title, teacher=teacher, 51 51 teacher_courses=int(teacher_courses), tag=tag, study_num=int(study_num), content=content) 52 52 course.save() 53 53 except Exception as e: 54 54 print('一条数据存取失败') 55 55 56 56 57 57 def get_course_link(url): 58 58 # 获取每一页的信息，传给下一个函数 59 59 response = requests.get(url) 60 60 xml = etree.HTML(response.text) 61 61 # contains()是包含的意思 62 62 courses = xml.xpath( 63 63 '//div[contains(@class, "col-md-3") and contains(@class, "col-sm-6") and contains(@class, "course")]') 64 64 for course in courses: 65 65 try: 66 66 url = 'https://www.shiyanlou.com' + course.xpath('.//a/@href')[0] 67 67 except Exception as e: 68 68 print('一个课程页面未获得') 69 69 continue 70 70 title = course.xpath('.//div[@class="course-name"]/text()')[0] 71 71 study_people = course.xpath( 72 72 './/span[@class="course-per-num pull-left"]/text()')[1].strip() 73 73 # study_people = int(study_people) 74 74 try: 75 75 tag = course.xpath( 76 76 './/span[@class="course-money pull-right"]/text()')[0] 77 77 except Exception as e: 78 78 tag = "普通" 79 79 parse_content(url=url, title=title, tag=tag, study_num=study_people) 80 80 # time.sleep(0.5) 81 81 82 82 83 83 def main(): 84 84 # 通过requests库的get获得目标地址的返回信息，类型为Response 85 85 response = requests.get('https://www.shiyanlou.com/courses/') 86 86 # 将返回信息的文本转化为xml树，可以通过xpath来进行查询 87 87 xml = etree.HTML(response.text) 88 88 # 由分析网页源代码可以总结，url分页模式，只有最后的数字不一样 89 89 course_link = 'https://www.shiyanlou.com/courses/?category=all&course_type=all&fee=all&tag=all&page={}' 90 90 # 这里获得最大页数就可以了,xpath()函数里的便是寻找路径了 91 91 # //会在全文来进行查找，//ul则是查找全文的ul标签，//ul[@class="pagination"]会仅查找有class属性， 92 92 # 且为"pagination"的标签，之后/li是查找当前的ul标签下的li标签（仅取一层),取查询到的列表倒数第二个标签 93 93 # 为li[last()-1],/a/text()查询a标签里的文本内容 94 94 page = xml.xpath('//ul[@class="pagination"]/li[last()-1]/a/text()') 95 95 if len(page) != 1: 96 96 print('爬取最大页数时发生错误！！') 97 97 return None 98 98 # page原是一个列表，这里取出它的元素，并转化为Int型 99 99 page = int(page[0]) 100100 # 将每一页的url传给get_course_link函数进行处理 101101 for i in range(1, page + 1): 102102 # 填入course_link,获取完整url 103103 url = course_link.format(i) 104104 print('页面地址：' + url) 105105 # 调用另一个函数 106106 get_course_link(url) 107107 108108 109109 if __name__ == '__main__': 110110 # 调用main函数 111111 main() 112112 113113 # [Finished in 218.5s]