C# 爬取图片

网络收集整理爬取图片

引用AngleSharp NuGet 包

1using AngleSharp;
2using System;
3using System.Collections.Generic;
4using System.IO;
5using System.Linq;
6using System.Net;
7
8namespace CoreConsoleApp
9{
10    public class Program
11    {
12        public static void Main(string[] args)
13        {
14            // 设置配置以支持文档加载
15            var config = Configuration.Default.WithDefaultLoader();
16            int pageIndex = 50;
17            for (int i = 1; i < pageIndex; i++)
18            {
19                // url地址
20                //var address = "https://www.qwe.com";
21                var address = @"https://www.qwe.com?pageIndex=" + i;
22                // 请求网页
23                var document = BrowsingContext.New(config).OpenAsync(address);
24                // 根据class获取html元素
25                var cells = document.Result.QuerySelectorAll(".panel-body li");
26                var fileName = (i + " - " + document.Result.Title).Replace("|", "");
27                foreach (var item in cells)
28                {
29                    //var belle = new Belle
30                    //{
31                    //    Title = item.QuerySelector("img").GetAttribute("title"),
32                    //    ImageUrl = item.QuerySelector("img").GetAttribute("src")
33                    //};
34                    var imageUrl = item.QuerySelector("img").GetAttribute("src");
35                    //string str = DateTime.Now.ToString("yyyyMMddHHmmss");
36                    string localPath = "D:\\Image\\" + DateTime.Now.ToString("yyyMMdd") + "\\" + fileName;
37                    List<string> arr = imageUrl.Split('/').ToList();
38                    if (!Directory.Exists(localPath))
39                    {
40                        // Create the directory it does not exist.
41                        Directory.CreateDirectory(localPath);
42                    }
43                    string filepath = localPath + "\\" + arr.Last();
44
45                    //方法一
46                    //Download(imageUrl, filepath);
47
48                    //方法二
49                    WebClient mywebclient = new WebClient();
50                    mywebclient.DownloadFile(imageUrl, filepath);
51                }
52            }
53
54            Console.ReadLine();
55        }
56
57        /// <summary>
58        /// Http方式下载文件
59        /// </summary>
60        /// <param name="url">http地址</param>
61        /// <param name="localfile">本地文件</param>
62        /// <returns></returns>
63        public static bool Download(string url, string localfile)
64        {
65            bool flag = false;
66            long startPosition = 0; // 上次下载的文件起始位置
67            FileStream writeStream; // 写入本地文件流对象
68
69            long remoteFileLength = GetHttpLength(url);// 取得远程文件长度
70            System.Console.WriteLine("remoteFileLength=" + remoteFileLength);
71            if (remoteFileLength == 745)
72            {
73                System.Console.WriteLine("远程文件不存在.");
74                return false;
75            }
76
77            // 判断要下载的文件夹是否存在
78            if (File.Exists(localfile))
79            {
80
81                writeStream = File.OpenWrite(localfile);             // 存在则打开要下载的文件
82                startPosition = writeStream.Length;                  // 获取已经下载的长度
83
84                if (startPosition >= remoteFileLength)
85                {
86                    System.Console.WriteLine("本地文件长度" + startPosition + "已经大于等于远程文件长度" + remoteFileLength);
87                    writeStream.Close();
88
89                    return false;
90                }
91                else
92                {
93                    writeStream.Seek(startPosition, SeekOrigin.Current); // 本地文件写入位置定位
94                }
95            }
96            else
97            {
98                writeStream = new FileStream(localfile, FileMode.Create);// 文件不保存创建一个文件
99                startPosition = 0;
100            }
101
102
103            try
104            {
105                HttpWebRequest myRequest = (HttpWebRequest)HttpWebRequest.Create(url);// 打开网络连接
106
107                if (startPosition > 0)
108                {
109                    myRequest.AddRange((int)startPosition);// 设置Range值,与上面的writeStream.Seek用意相同,是为了定义远程文件读取位置
110                }
111
112
113                Stream readStream = myRequest.GetResponse().GetResponseStream();// 向服务器请求,获得服务器的回应数据流
114
115
116                byte[] btArray = new byte[512];// 定义一个字节数据,用来向readStream读取内容和向writeStream写入内容
117                int contentSize = readStream.Read(btArray, 0, btArray.Length);// 向远程文件读第一次
118
119                long currPostion = startPosition;
120
121                while (contentSize > 0)// 如果读取长度大于零则继续读
122                {
123                    currPostion += contentSize;
124                    int percent = (int)(currPostion * 100 / remoteFileLength);
125                    System.Console.WriteLine("percent=" + percent + "%");
126
127                    writeStream.Write(btArray, 0, contentSize);// 写入本地文件
128                    contentSize = readStream.Read(btArray, 0, btArray.Length);// 继续向远程文件读取
129                }
130
131                //关闭流
132                writeStream.Close();
133                readStream.Close();
134
135                flag = true;        //返回true下载成功
136            }
137            catch (Exception)
138            {
139                writeStream.Close();
140                flag = false;       //返回false下载失败
141            }
142
143            return flag;
144        }
145
146        // 从文件头得到远程文件的长度
147        private static long GetHttpLength(string url)
148        {
149            long length = 0;
150
151            try
152            {
153                HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);// 打开网络连接
154                HttpWebResponse rsp = (HttpWebResponse)req.GetResponse();
155
156                if (rsp.StatusCode == HttpStatusCode.OK)
157                {
158                    length = rsp.ContentLength;// 从文件头得到远程文件的长度
159                }
160
161                rsp.Close();
162                return length;
163            }
164            catch (Exception e)
165            {
166                return length;
167            }
168
169        }
170
171    }
172}

评论区

相关推荐