2018/05/29 修改抓取编码gb2312改gb18030
项目需要行政区域三级联动，刚好写个爬虫练练手。
Laravel 框架，安装的两个库
1 composer require guzzlehttp/guzzle 2 composer require symfony/dom-crawler
创建表
1DROP TABLE IF EXISTS `area`; 2CREATE TABLE `area` ( 3 `id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, 4 `name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, 5 `parent_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, 6 PRIMARY KEY (`id`) 7) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; 8 9DROP TABLE IF EXISTS `crawler`; 10CREATE TABLE `crawler` ( 11 `id` int(11) NOT NULL AUTO_INCREMENT, 12 `status` int(11) DEFAULT '0', 13 `data` text COLLATE utf8mb4_unicode_ci, 14 PRIMARY KEY (`id`) 15) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
创建对应Model
App\Model\Area.php
1<?php 2 3namespace App\Model; 4 5use Illuminate\Database\Eloquent\Model; 6 7class Area extends Model 8{ 9 public $timestamps = false; 10 11 protected $table = 'area'; 12 13 protected $keyType = 'string'; 14 15 protected $fillable = [ 16 'id', 'name', 'parent_id', 17 ]; 18}
App\Model\Crawler.php
1<?php 2 3namespace App\Model; 4 5use Illuminate\Database\Eloquent\Model; 6 7class Crawler extends Model 8{ 9 public $timestamps = false; 10 11 protected $table = 'crawler'; 12 13 protected $fillable = [ 14 'id', 'status', 'data' 15 ]; 16}
app/Console/Kernel.php 添加
1 protected $commands = [ 2 'App\Console\Commands\CityCrawler', 3 ];
新建 App\Console\Commands\CityCrawler.php
1<?php 2namespace App\Console\Commands; 3 4use Illuminate\Console\Command; 5use Symfony\Component\DomCrawler\Crawler; 6 7use App\Model\Area; 8use App\Model\Crawler as CrawlerTask; 9 10use GuzzleHttp\Psr7; 11use GuzzleHttp\Exception\RequestException; 12 13// 流程： 14// 1. func top 抓取行政区域省级, 每个省链接生成一次抓取任务，保存到任务表crawler。 15// 2. 循环抓取 16// 1). 读取一条任务 17 select * from crawler where status = 0 limit 1; 18 update crawler set status = 1 where id = 本次任务id; 19// 2). 根据任务类型调用不同抓取方法 如： 20 镇级：crawler_towntr 21 区级：crawler_districts 22 城市：crawler_citys。 23// 保存抓取到数据到area表，并生成子级区域抓取任务。 24class CityCrawler extends Command 25{ 26 27 protected $signature = 'city:crawler'; 28 protected $description = 'City Crawler'; 29 protected $start_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/'; 30 protected $special_city = ['东莞市','中山市','嘉峪关市','三沙市','儋州市']; // 中国5个不设市辖区的地级市 31 32 public function handle() 33 { 34 //抓取省级行政区域 35 $this->top(); 36 37 while (true) { 38 $task_model = $this->task(); 39 40 if(empty($task_model)){ 41 return $this->info("End"); 42 } 43 $task= json_decode($task_model->data, true); 44 45 // 打印日志 46 $this->info(implode(',', array_map(function($item){ 47 return $item['id'] . ' ' . $item['name']; 48 }, $task['data']))); 49 50 $status = call_user_func(array($this, 'crawler_' . $task['crawler']), $task); 51 if($status){ 52 $this->finish($task_model); 53 }else{ 54 var_dump($task, 'error'); 55 return false; 56 } 57 $this->info("sleep 1"); 58 sleep(1); 59 } 60 } 61 62 63 public function finish($task){ 64 $task->status = 2; 65 $task->save(); 66 } 67 68 public function task($status = 0) 69 { 70 $task = CrawlerTask::where("status", $status)->first(); 71 $task->status = 1; // 进行中 72 $task->save(); 73 return $task; 74 } 75 76 public function push($data) 77 { 78 $task = new CrawlerTask; 79 $task->data = json_encode($data); 80 $task->save(); 81 } 82 83 // 第一个页面 84 public function top() 85 { 86 $url = $this->start_url; 87 88 $html = $this->send_http($url); 89 90 $crawler = new Crawler(); 91 $crawler->addHtmlContent($html, 'gb18030'); 92 $crawler->filter('.provincetr')->filter('td > a')->each(function(Crawler $node, $i) use($url) { 93 $text = $node->text(); 94 $href = $node->attr('href'); 95 $id = str_replace('.html', '', $href); 96 97 $task = [ 98 'crawler' => 'citys', 99 'remark' => '省', 100 'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href, 101 'data' => [ [ 'name' => $text, 'id' => $id] ], 102 'parent_id' => $id, 103 ]; 104 $this->push($task); 105 106 Area::create( 107 [ 108 'id' => $id, 109 'name' => $text, 110 'parent_id' => 0 111 ] 112 ); 113 114 $this->info($node->attr('href')); 115 $this->info($text); 116 }); 117 } 118 119 120 public function crawler_towntr($task) 121 { 122 123 $url = $task['url']; 124 125 if(!strpos($url, '.html')){ 126 $this->info('为空的直辖市'); 127 return true; 128 } 129 130 $html = $this->send_http($url); 131 132 $crawler = new Crawler(); 133 $crawler->addHtmlContent($html, 'gb18030'); 134 $crawler->filter('.towntr')->each(function(Crawler $node, $i) use ($task, $url) { 135 136 $code_node = $node->filter('td')->eq(0)->filter('a'); 137 $name_node = $node->filter('td')->eq(1)->filter('a'); 138 139 Area::create( 140 [ 141 'id' => $code_node->text(), 142 'name' => $name_node->text(), 143 'parent_id' => $task['parent_id'] 144 ] 145 ); 146 147 $this->info($code_node->text() . ' ' . $name_node->text()); 148 }); 149 150 return true; 151 } 152 153 public function crawler_districts($task) 154 { 155 $url = $task['url']; 156 $html = $this->send_http($url); 157 158 $crawler = new Crawler(); 159 $crawler->addHtmlContent($html, 'gb18030'); 160 $crawler->filter('.countytr')->each(function(Crawler $node, $i) use ($task, $url) { 161 162 163 $code_node = $node->filter('td')->eq(0)->filter('a'); 164 $name_node = $node->filter('td')->eq(1)->filter('a'); 165 166 //没有子节点 167 if($code_node->count() == 0){ 168 $code_node = $node->filter('td')->eq(0); 169 $name_node = $node->filter('td')->eq(1); 170 }else{ 171 172 $href = $code_node->attr("href"); 173 $data = $task['data']; 174 $data[] = ['name' => $name_node->text(), 'id' => $code_node->text()] ; 175 176 $new_task = [ 177 'crawler' => 'towntr', 178 'remark' => '县 区', 179 'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href, 180 'data' => $data, 181 'parent_id' => $code_node->text(), 182 ]; 183 184 $this->push($new_task); 185 } 186 187 $this->info($code_node->text() . ' ' . $name_node->text()); 188 189 Area::create( 190 [ 191 'id' => $code_node->text(), 192 'name' => $name_node->text(), 193 'parent_id' => $task['parent_id'] 194 ] 195 ); 196 197 }); 198 return true; 199 } 200 201 public function crawler_citys($task) 202 { 203 $url = $task['url']; 204 $html = $this->send_http($url); 205 206 $crawler = new Crawler(); 207 $crawler->addHtmlContent($html, 'gb18030'); 208 $crawler->filter('.citytr')->each(function(Crawler $node, $i) use ($task, $url) { 209 210 $code_node = $node->filter('td')->eq(0)->filter('a'); 211 $name_node = $node->filter('td')->eq(1)->filter('a'); 212 $href = $code_node->attr("href"); 213 214 $this->info($code_node->text() . ' ' . $name_node->text()); 215 216 Area::create( 217 [ 218 'id' => $code_node->text(), 219 'name' => $name_node->text(), 220 'parent_id' => $task['parent_id'] 221 ] 222 ); 223 224 225 $data = $task['data']; 226 $data[] = ['name' => $name_node->text(), 'id' => $code_node->text()] ; 227 228 if(in_array($name_node->text(), $this->special_city)){ 229 $new_task = [ 230 'crawler' => 'towntr', 231 'remark' => '特别的5个省地级市', 232 'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href, 233 'data' => $data, 234 'parent_id' => $code_node->text(), 235 ]; 236 }else{ 237 238 $new_task = [ 239 'crawler' => 'districts', 240 'remark' => '城市', 241 'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href, 242 'data' => $data, 243 'parent_id' => $code_node->text(), 244 ]; 245 246 } 247 248 249 $this->push($new_task); 250 251 }); 252 253 return true; 254 } 255 256 public function info($string, $verbosity = null) 257 { 258 $string = iconv( 'UTF-8', 'GB18030', $string); // cmd 中文gbk编码 259 parent::line($string, 'info', $verbosity); 260 } 261 262 private function send_http($url) 263 { 264 $user_agent_list = [ 265 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.04', 266 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.9 Safari/537.36', 267 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52', 268 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0', 269 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27', 270 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko', 271 ]; 272 273 $user_agent = $user_agent_list[(time() % 6)]; 274 $timeout = 5; // 秒 275 276 $client = new \GuzzleHttp\Client(['headers' => ['User-Agent' => $user_agent], 'timeout' => $timeout]); 277 278 try { 279 $res = $client->request('GET', $url); 280 $html = (string)$res->getBody(); 281 282 } catch (RequestException $e) { 283 // 抓取中会有404状态返回，再重新请求一次。 284 $this->info(Psr7\str($e->getRequest())); 285 if ($e->hasResponse()) { 286 $this->info(Psr7\str($e->getResponse())); 287 } 288 289 $this->info("send_http timeout retry"); 290 $this->info("sleep 2s"); 291 sleep(2); 292 $res = $client->request('GET', $url); 293 $html = (string)$res->getBody(); 294 } 295 296 return $html; 297 } 298 299}
进目录运行
php artisan city:crawler
最后
------------------------------------------------------
数据有了全部写到一个json文件里，太大了1M多 :(
还是写成ajax从服务端读取三级联动数据。
area.js
1// 1. 省加载 其他请选择 2// 2. 省 change 触发加载市 3// 3. 市触发加载区 4// 4. 区加载触发街道 5function area_init(param) { 6 var area = this; 7 area.area_not_filter = false;; 8 area.prompt_html = '<option value="">－请选择－</option>'; 9 if (param.area_not_filter) { 10 area.area_not_filter = param.area_not_filter; 11 } 12 if (param.province) { 13 area.province_el = $(param.province); 14 } 15 if (param.city) { 16 area.city_el = $(param.city); 17 } 18 if (param.district) { 19 area.district_el = $(param.district); 20 } 21 if (param.street) { 22 area.street_el = $(param.street); 23 } 24 area.load = function() { 25 area.province_el.html(area.prompt_html); 26 area.city_el.html(area.prompt_html); 27 area.district_el.html(area.prompt_html); 28 if (area.street_el) { 29 area.street_el.html(area.prompt_html); 30 } 31 province_id = area.province_el.attr("data-value"); 32 city_id = area.city_el.attr("data-value"); 33 district_id = area.district_el.attr("data-value"); 34 if (area.street_el) { 35 street_id = area.street_el.attr("data-value"); 36 } 37 area.area_fill(0, 'province', area.province_el, province_id); 38 province_id && area.area_fill(province_id, 'city', area.city_el, city_id); 39 city_id && area.area_fill(city_id, 'district', area.district_el, district_id); 40 if (area.street_el) { 41 district_id && area.area_fill(district_id, 'street', area.street_el, street_id); 42 } 43 } 44 area.bind = function() { 45 area.province_el.change(function() { 46 area.area_fill($(this).val(), 'city', area.city_el); 47 area.city_el.html(area.prompt_html); 48 area.district_el.html(area.prompt_html); 49 if (area.street_el) { 50 area.street_el.html(area.prompt_html); 51 } 52 }); 53 area.city_el.change(function() { 54 area.area_fill($(this).val(), 'district', area.district_el); 55 area.district_el.html(area.prompt_html); 56 if (area.street_el) { 57 area.street_el.html(area.prompt_html); 58 } 59 }); 60 if (area.street_el) { 61 area.district_el.change(function() { 62 area.area_fill($(this).val(), 'street', area.street_el); 63 area.street_el.html(area.prompt_html); 64 }); 65 } 66 }; 67 area.area_fill = function(parent_id, level, el, active_id) { 68 // value='' 不请求 ajax 69 if (parent_id === '') { 70 return false; 71 } 72 area.get_area(parent_id, level, function(list) { 73 var province = area.prompt_html; 74 $.each(list, function(i, n) { 75 province += '<option ' + (active_id == n.id ? ' selected ' : ' ') + ' value="' + n.id + '">' + n.name + '</option>'; 76 }); 77 $(el).html(province); 78 }); 79 } 80 area.get_area = function(parent_id, level, callback) { 81 var $url = '/ajax_area/' + level + '/' + parent_id; 82 $.ajax({ 83 url: $url, 84 type: 'get', 85 success: function(res) { 86 callback(res); 87 } 88 }); 89 } 90 area.load(); 91 area.bind(); 92} 93 94// 使用方法 95// data-value 默认值 96// 32 江苏省 97// 320100000000 南京市 98// 320102000000 玄武区 99// <select name="province" data-value="32"></select> 100// <select name="city" data-value="320100000000"></select> 101// <select name="district" data-value="320102000000"></select> 102// <select name="street" data-value=""></select> 103 104// var area = new area_init( 105// { 106// province: "select[name='province']", 107// city: "select[name='city']", 108// district: "select[name='district']", 109// street: "select[name='street']" 110// } 111// ); 112 113// 或者 114 115// <select name="province" data-value="32"></select> 116// <select name="city" data-value="320100000000"></select> 117 118// var area = new area_init( 119// { 120// province: "select[name='province']", 121// city: "select[name='city']" 122// } 123// ); 124 125// 如果需要js动态修改 126// $('select[name="province"]').attr('data-value', data.province); 127// $('select[name="city"]').attr('data-value', data.city); 128// $('select[name="district"]').attr('data-value', data.district); 129// area.load();
成品长这样哈
