# 爬取有关软件工作的信息
1import requests 2from pymysql import connect 3from bs4 import BeautifulSoup 4 5# 定义数据库的连接函数 6conn = connect(user="root", password="root", host="localhost", database="python", charset="utf8") 7cursor = conn.cursor() 8 9 10# 获取工作数据 11def get_html_resources(url, headers): 12 response = requests.get(url, headers=headers) 13 if response.status_code == 200: 14 return response.text 15 else: 16 print("获取网站源码出错........") 17# 解析网站源码 18def parse_detail_page(html, headers, table, cursor): 19 parttern = re.compile('engine_search_result":(.*?)</script>', re.S) 20 items = re.findall(parttern, html) 21 parttern_detail = re.compile( 22 'job_href":"(.*?)","job_name":"(.*?)".*?company_name":"(.*?)","providesalary_text":"(.*?)".*?attribute_text":(.*?),"companysize_text', 23 re.S) 24 items_detail = re.findall(parttern_detail, items[0]) 25 for item in items_detail: 26 address = [] 27 education = [] 28 content = [] 29 job_url = str(item[0]).replace("\\", "") 30 job_address_education = str(item[4]).replace('["', "").replace('"]', "").replace('"', "").split(",") 31 if len(job_address_education) == 4: 32 address.append(job_address_education[0]) 33 education.append(job_address_education[2]) 34 if len(job_address_education) == 3: 35 address.append(job_address_education[0]) 36 education.append(job_address_education[1]) 37 38 # 开始获取详情页的工作数据 39 response = requests.get(job_url, headers=headers) 40 response.encoding = "gbk" 41 try: 42 if response.status_code == 200: 43 detail_html = response.text 44 soup = BeautifulSoup(detail_html, "lxml") 45 job_request = soup.find("div", class_="bmsg job_msg inbox").text 46 content.append(job_request) 47 else: 48 print("获取详情页的信息错误") 49 pass 50 except: 51 pass 52 53 yield { 54 "工作名称": item[1], 55 "公司名称": item[2], 56 "工作待遇": item[3], 57 "工作地点": address[0], 58 "学历要求": education[0], 59 "工作要求": content[0], 60 } 61 try: 62 sql = "insert into " + str( 63 table) + "(job_name,company_name,salary,job_address,education,job_require) values ('" + item[ 64 1] + "','" + item[2] + "','" + item[3] + "','" + address[0] + "','" + education[0] + "','" + \ 65 content[0] + "');" 66 cursor.execute(sql) 67 conn.commit() 68 except: 69 print("数据插入异常............") 70 conn.rollback() 71 72 return items 73 74 75def main(): 76 # 创建headers信息 77 headers = { 78 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36" 79 } 80 81 # 创建数据表列表 82 table_list = ["android", "nature_language", "deep_learning", "computer_vision", "big_data", "machine_learning", 83 "production_manager", "education", "finance", "service", "transportation"] 84 job_name_list = ["android开发", "自然语言处理", "深度学习", "计算机视觉", "大数据", "机器学习", "产品经理", "教师", "金融", "服务类", "运输"] 85 # 创建数据库的sql语句 86 # 定义sql语句用来创建表结构 87 for table in table_list: 88 try: 89 sql = "create table " + str( 90 table) + "(job_name varchar(200),company_name varchar(300),salary varchar(300),job_address varchar(500),education varchar(100),job_require varchar(5000),min_salary int(11) null,max_salary int(11));" 91 # 使用cursor执行sql语句进行表结构的创建 92 cursor.execute(sql) 93 conn.commit() 94 95 except: 96 print("数据表已存在正......................") 97 pass 98 99 finally: 100 # 设置关键字开始抓取工作数据 101 job = job_name_list[table_list.index(table)] 102 for i in range(2): 103 url = "https://search.51job.com/list/000000,000000,0000,00,9,99," + str(job) + ",2," + str( 104 i + 1) + ".html?lang=c&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=" 105 try: 106 html = get_html_resources(url, headers) 107 items = parse_detail_page(html, headers, table, cursor) 108 for item in items: 109 print(item) 110 except: 111 print(url) 112 print("获取异常") 113 pass 114 115 116if __name__ == "__main__": 117 main() 118