版权所有，转载请注明：http://www.lenggirl.com/language/go-picture.html
使用准备
1.安装Golang
2.下载爬虫包
1go get -v github.com/hunterhug/marmot/expert 2go get -v github.com/hunterhug/marmot/miner 3go get -v github.com/hunterhug/parrot/util
程序
该程序只能抓取HTML中src="http"中的图片, 必须带有协议头http(s), 其他如data-src和混淆在JS中的无法抓取
See: https://github.com/hunterhug/marmot/blob/master/example/lesson/lesson6.go
1package main 2 3import ( 4 "errors" 5 "fmt" 6 "net/url" 7 "strings" 8 9 "github.com/hunterhug/marmot/expert" 10 "github.com/hunterhug/marmot/miner" 11 "github.com/hunterhug/parrot/util" 12) 13 14// Num of miner, We can run it at the same time to crawl data fast 15var MinerNum = 5 16 17// You can update this decide whether to proxy 18var ProxyAddress interface{} 19 20func main() { 21 // You can Proxy! 22 // ProxyAddress = "socks5://127.0.0.1:1080" 23 24 fmt.Println(`Welcome: Input "url" and picture keep "dir"`) 25 for { 26 fmt.Println("---------------------------------------------") 27 url := util.Input(`URL(Like: "http://publicdomainarchive.com")`, "http://publicdomainarchive.com") 28 dir := util.Input(`DIR(Default: "./picture")`, "./picture") 29 fmt.Printf("You will keep %s picture in dir %s\n", url, dir) 30 fmt.Println("---------------------------------------------") 31 32 // Start Catch 33 err := CatchPicture(url, dir) 34 if err != nil { 35 fmt.Println("Error:" + err.Error()) 36 } 37 } 38} 39 40// Come on! 41func CatchPicture(picture_url string, dir string) error { 42 // Check valid 43 _, err := url.Parse(picture_url) 44 if err != nil { 45 return err 46 } 47 48 // Make dir! 49 err = util.MakeDir(dir) 50 if err != nil { 51 return err 52 } 53 54 // New a worker to get url 55 worker, _ := miner.New(ProxyAddress) 56 57 result, err := worker.SetUrl(picture_url).SetUa(miner.RandomUa()).Get() 58 if err != nil { 59 return err 60 } 61 62 // Find all picture 63 pictures := expert.FindPicture(string(result)) 64 65 // Empty, What a pity! 66 if len(pictures) == 0 { 67 return errors.New("empty") 68 } 69 70 // Devide pictures into several worker 71 xxx, _ := util.DevideStringList(pictures, MinerNum) 72 73 // Chanel to info exchange 74 chs := make(chan int, len(pictures)) 75 76 // Go at the same time 77 for num, imgs := range xxx { 78 79 // Get pool miner 80 worker_picture, ok := miner.Pool.Get(util.IS(num)) 81 if !ok { 82 // No? set one! 83 worker_temp, _ := miner.New(ProxyAddress) 84 worker_picture = worker_temp 85 worker_temp.SetUa(miner.RandomUa()) 86 miner.Pool.Set(util.IS(num), worker_temp) 87 } 88 89 // Go save picture! 90 go func(imgs []string, worker *miner.Worker, num int) { 91 for _, img := range imgs { 92 93 // Check, May be Pass 94 _, err := url.Parse(img) 95 if err != nil { 96 continue 97 } 98 99 // Change Name of our picture 100 filename := strings.Replace(util.ValidFileName(img), "#", "_", -1) 101 102 // Exist? 103 if util.FileExist(dir + "/" + filename) { 104 fmt.Println("File Exist：" + dir + "/" + filename) 105 chs <- 0 106 } else { 107 108 // Not Exsit? 109 imgsrc, e := worker.SetUrl(img).Get() 110 if e != nil { 111 fmt.Println("Download " + img + " error:" + e.Error()) 112 chs <- 0 113 return 114 } 115 116 // Save it! 117 e = util.SaveToFile(dir+"/"+filename, imgsrc) 118 if e == nil { 119 fmt.Printf("SP%d: Keep in %s/%s\n", num, dir, filename) 120 } 121 chs <- 1 122 } 123 } 124 }(imgs, worker_picture, num) 125 } 126 127 // Every picture should return 128 for i := 0; i < len(pictures); i++ { 129 <-chs 130 } 131 132 return nil 133}
解释均写, 运行后:
1superpika@superpika-chen-110:~/code/src/github.com/hunterhug/marmot/example/lesson$ go run lesson6.go 2 3 Welcome: Input "url" and picture keep "dir" 4 5 6 7--------------------------------------------- 8URL(Like: "http://publicdomainarchive.com") 9 10DIR(Default: "./picture") 11 12You will keep http://publicdomainarchive.com picture in dir ./picture 13--------------------------------------------- 14SP0: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_02_03_modern.jpg 15SP4: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_03_03_google_dark.png 16SP2: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-003-667x1000-192684_667x675.jpg 17 18superpika@superpika-chen-110:~/code/src/github.com/hunterhug/marmot/example/lesson$ go run lesson6.go 19 20 Welcome: Input "url" and picture keep "dir" 21 22 23 24--------------------------------------------- 25URL(Like: "http://publicdomainarchive.com") 26 27DIR(Default: "./picture") 28 29You will keep http://publicdomainarchive.com picture in dir ./picture 30--------------------------------------------- 31SP0: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_02_03_modern.jpg 32SP4: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_03_03_google_dark.png 33SP2: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-003-667x1000-192684_667x675.jpg 34SP4: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_05_03_powered-by-wp-engine.png 35SP4: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_05_03_divi.png 36SP2: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-002-1000x667.jpg 37SP4: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_05_03_public-domain-mark.png 38SP3: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_01_03_public-domain-images-free-stock-photos008-1000x625.jpg 39SP0: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_09_03_Weekly.jpg 40SP1: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_11_03_free-stock-photos-public-domain-images-054-1000x667.jpg 41SP3: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_10_03_instagram_dark.png 42SP0: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_02_03_vintage.jpg 43SP2: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-001-1000x667.jpg 44SP1: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_11_03_free-stock-photos-public-domain-images-070-1000x667.jpg 45SP3: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_03_03_twitter02_dark.png 46SP2: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_01_03_public-domain-images-free-stock-photos001-1000x750-167066_1000x675.jpg 47SP1: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-035-1000x667.jpg 48SP3: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_03_03_facebook_dark.png 49SP0: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_11_03_free-stock-photos-public-domain-images-060-1000x667.jpg 50SP1: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-013-1000x667.jpg 51--------------------------------------------- 52URL(Like: "http://publicdomainarchive.com") 53 54 55 56SP4: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_05_03_powered-by-wp-engine.png 57SP4: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_05_03_divi.png 58SP2: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-002-1000x667.jpg 59SP4: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_05_03_public-domain-mark.png 60SP3: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_01_03_public-domain-images-free-stock-photos008-1000x625.jpg 61SP0: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_09_03_Weekly.jpg 62SP1: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_11_03_free-stock-photos-public-domain-images-054-1000x667.jpg 63SP3: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_10_03_instagram_dark.png 64SP0: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_02_03_vintage.jpg 65SP2: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-001-1000x667.jpg 66SP1: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_11_03_free-stock-photos-public-domain-images-070-1000x667.jpg 67SP3: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_03_03_twitter02_dark.png 68SP2: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_01_03_public-domain-images-free-stock-photos001-1000x750-167066_1000x675.jpg 69SP1: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-035-1000x667.jpg 70SP3: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2014_03_03_03_facebook_dark.png 71SP0: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_11_03_free-stock-photos-public-domain-images-060-1000x667.jpg 72SP1: Keep in ./picture/http_04__03__03_publicdomainarchive.com_03_wp-content_03_uploads_03_2017_03_09_03_free-stock-photos-public-domain-images-013-1000x667.jpg 73--------------------------------------------- 74URL(Like: "http://publicdomainarchive.com")
[图片上传失败...(image-685ed8-1557639583356)]