转自：http://blog.csdn.net/huangxy10/article/details/8120106
备注：把项目属性中的字符集改成多字节集合?
1 // 网络爬虫.cpp : 定义控制台应用程序的入口点。
2 //
3
4 #include "stdafx.h"
5 /*
6
7 int _tmain(int argc, _TCHAR* argv[])
8 {
9 return 0;
10 }
11
12 */
13
14 //#include <Windows.h>
15 #include <string>
16 #include <iostream>
17 #include <fstream>
18 #include <vector>
19 #include "winsock2.h"
20 #include <time.h>
21 #include <queue>
22 #include <hash_set>
23
24 #pragma comment(lib, "ws2_32.lib")
25 using namespace std;
26
27 #define DEFAULT_PAGE_BUF_SIZE 1048576
28
29 queue<string> hrefUrl;
30 hash_set<string> visitedUrl;
31 hash_set<string> visitedImg;
32 int depth=0;
33 int g_ImgCnt=1;
34
35 //解析URL，解析出主机名，资源名
36 bool ParseURL( const string & url, string & host, string & resource){
37 if ( strlen(url.c_str()) > 2000 ) {
38 return false;
39 }
40
41 const char * pos = strstr( url.c_str(), "http://" );
42 if( pos==NULL ) pos = url.c_str();
43 else pos += strlen("http://");
44 if( strstr( pos, "/")==0 )
45 return false;
46 char pHost[100];
47 char pResource[2000];
48 sscanf( pos, "%[^/]%s", pHost, pResource );
49 host = pHost;
50 resource = pResource;
51 return true;
52 }
53
54 //使用Get请求，得到响应
55 bool GetHttpResponse( const string & url, char * &response, int &bytesRead ){
56 string host, resource;
57 if(!ParseURL( url, host, resource )){
58 cout << "Can not parse the url"<<endl;
59 return false;
60 }
61
62 //建立socket
63 struct hostent * hp= gethostbyname( host.c_str() );
64 if( hp==NULL ){
65 cout<< "Can not find host address"<<endl;
66 return false;
67 }
68
69 SOCKET sock = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP);
70 if( sock == -1 || sock == -2 ){
71 cout << "Can not create sock."<<endl;
72 return false;
73 }
74
75 //建立服务器地址
76 SOCKADDR_IN sa;
77 sa.sin_family = AF_INET;
78 sa.sin_port = htons( 80 );
79 //char addr[5];
80 //memcpy( addr, hp->h_addr, 4 );
81 //sa.sin_addr.s_addr = inet_addr(hp->h_addr);
82 memcpy( &sa.sin_addr, hp->h_addr, 4 );
83
84 //建立连接
85 if( 0!= connect( sock, (SOCKADDR*)&sa, sizeof(sa) ) ){
86 cout << "Can not connect: "<< url <<endl;
87 closesocket(sock);
88 return false;
89 };
90
91 //准备发送数据
92 string request = "GET " + resource + " HTTP/1.1\r\nHost:" + host + "\r\nConnection:Close\r\n\r\n";
93
94 //发送数据
95 if( SOCKET_ERROR ==send( sock, request.c_str(), request.size(), 0 ) ){
96 cout << "send error" <<endl;
97 closesocket( sock );
98 return false;
99 }
100
101 //接收数据
102 int m_nContentLength = DEFAULT_PAGE_BUF_SIZE;
103 char *pageBuf = (char *)malloc(m_nContentLength);
104 memset(pageBuf, 0, m_nContentLength);
105
106 bytesRead = 0;
107 int ret = 1;
108 cout <<"Read: ";
109 while(ret > 0){
110 ret = recv(sock, pageBuf + bytesRead, m_nContentLength - bytesRead, 0);
111
112 if(ret > 0)
113 {
114 bytesRead += ret;
115 }
116
117 if( m_nContentLength - bytesRead<100){
118 cout << "\nRealloc memorry"<<endl;
119 m_nContentLength *=2;
120 pageBuf = (char*)realloc( pageBuf, m_nContentLength); //重新分配内存
121 }
122 cout << ret <<" ";
123 }
124 cout <<endl;
125
126 pageBuf[bytesRead] = '\0';
127 response = pageBuf;
128 closesocket( sock );
129 return true;
130 //cout<< response <<endl;
131 }
132
133 //提取所有的URL以及图片URL
134 void HTMLParse ( string & htmlResponse, vector<string> & imgurls, const string & host ){
135 //找所有连接，加入queue中
136 const char *p= htmlResponse.c_str();
137 char *tag="href=\"";
138 const char *pos = strstr( p, tag );
139 ofstream ofile("url.txt", ios::app);
140 while( pos ){
141 pos +=strlen(tag);
142 const char * nextQ = strstr( pos, "\"" );
143 if( nextQ ){
144 char * url = new char[ nextQ-pos+1 ];
145 //char url[100]; //固定大小的会发生缓冲区溢出的危险
146 sscanf( pos, "%[^\"]", url);
147 string surl = url; // 转换成string类型，可以自动释放内存
148 if( visitedUrl.find( surl ) == visitedUrl.end() ){
149 visitedUrl.insert( surl );
150 ofile << surl<<endl;
151 hrefUrl.push( surl );
152 }
153 pos = strstr(pos, tag );
154 delete [] url; // 释放掉申请的内存
155 }
156 }
157 ofile << endl << endl;
158 ofile.close();
159
160 tag ="<img ";
161 const char* att1= "src=\"";
162 const char* att2="lazy-src=\"";
163 const char *pos0 = strstr( p, tag );
164 while( pos0 ){
165 pos0 += strlen( tag );
166 const char* pos2 = strstr( pos0, att2 );
167 if( !pos2 || pos2 > strstr( pos0, ">") ) {
168 pos = strstr( pos0, att1);
169 if(!pos) {
170 pos0 = strstr(att1, tag );
171 continue;
172 } else {
173 pos = pos + strlen(att1);
174 }
175 }
176 else {
177 pos = pos2 + strlen(att2);
178 }
179
180 const char * nextQ = strstr( pos, "\"");
181 if( nextQ ){
182 char * url = new char[nextQ-pos+1];
183 sscanf( pos, "%[^\"]", url);
184 cout << url<<endl;
185 string imgUrl = url;
186 if( visitedImg.find( imgUrl ) == visitedImg.end() ){
187 visitedImg.insert( imgUrl );
188 imgurls.push_back( imgUrl );
189 }
190 pos0 = strstr(pos0, tag );
191 delete [] url;
192 }
193 }
194 cout << "end of Parse this html"<<endl;
195 }
196
197 //把URL转化为文件名
198 string ToFileName( const string &url ){
199 string fileName;
200 fileName.resize( url.size());
201 int k=0;
202 for( int i=0; i<(int)url.size(); i++){
203 char ch = url[i];
204 if( ch!='\\'&&ch!='/'&&ch!=':'&&ch!='*'&&ch!='?'&&ch!='"'&&ch!='<'&&ch!='>'&&ch!='|')
205 fileName[k++]=ch;
206 }
207 return fileName.substr(0,k) + ".txt";
208 }
209
210 //下载图片到img文件夹
211 void DownLoadImg( vector<string> & imgurls, const string &url ){
212
213 //生成保存该url下图片的文件夹
214 string foldname = ToFileName( url );
215 foldname = "./img/"+foldname;
216 if(!CreateDirectory( (LPCSTR)foldname.c_str(),NULL ))
217 cout << "Can not create directory:"<< foldname<<endl;
218 char *image;
219 int byteRead;
220 for( int i=0; i<imgurls.size(); i++){
221 //判断是否为图片，bmp，jgp，jpeg，gif
222 string str = imgurls[i];
223 int pos = str.find_last_of(".");
224 if( pos == string::npos )
225 continue;
226 else{
227 string ext = str.substr( pos+1, str.size()-pos-1 );
228 if( ext!="bmp"&& ext!="jpg" && ext!="jpeg"&& ext!="gif"&&ext!="png")
229 continue;
230 }
231 //下载其中的内容
232 if( GetHttpResponse(imgurls[i], image, byteRead)){
233 if ( strlen(image) ==0 ) {
234 continue;
235 }
236 const char *p=image;
237 const char * pos = strstr(p,"\r\n\r\n")+strlen("\r\n\r\n");
238 int index = imgurls[i].find_last_of("/");
239 if( index!=string::npos ){
240 string imgname = imgurls[i].substr( index , imgurls[i].size() );
241 ofstream ofile( foldname+imgname, ios::binary );
242 if( !ofile.is_open() )
243 continue;
244 cout <<g_ImgCnt++<< foldname+imgname<<endl;
245 ofile.write( pos, byteRead- (pos-p) );
246 ofile.close();
247 }
248 free(image);
249 }
250 }
251 }
252
253
254
255 //广度遍历
256 void BFS( const string & url ){
257 char * response;
258 int bytes;
259 // 获取网页的相应，放入response中。
260 if( !GetHttpResponse( url, response, bytes ) ){
261 cout << "The url is wrong! ignore." << endl;
262 return;
263 }
264 string httpResponse=response;
265 free( response );
266 string filename = ToFileName( url );
267 ofstream ofile( "./html/"+filename );
268 if( ofile.is_open() ){
269 // 保存该网页的文本内容
270 ofile << httpResponse << endl;
271 ofile.close();
272 }
273 vector<string> imgurls;
274 //解析该网页的所有图片链接，放入imgurls里面
275 HTMLParse( httpResponse, imgurls, url );
276
277 //下载所有的图片资源
278 DownLoadImg( imgurls, url );
279 }
280
281 void main()
282 {
283 //初始化socket，用于tcp网络连接
284 WSADATA wsaData;
285 if( WSAStartup(MAKEWORD(2,2), &wsaData) != 0 ){
286 return;
287 }
288
289 // 创建文件夹，保存图片和网页文本文件
290 CreateDirectory((LPCSTR) "./img",0);
291 CreateDirectory((LPCSTR)"./html",0);
292 //string urlStart = "http://hao.360.cn/meinvdaohang.html";
293
294 // 遍历的起始地址
295 // string urlStart = "http://www.wmpic.me/tupian";
296 string urlStart = "http://item.taobao.com/item.htm?spm=a230r.1.14.19.sBBNbz&id=36366887850&ns=1#detail";
297
298 // 使用广度遍历
299 // 提取网页中的超链接放入hrefUrl中，提取图片链接，下载图片。
300 BFS( urlStart );
301
302 // 访问过的网址保存起来
303 visitedUrl.insert( urlStart );
304
305 while( hrefUrl.size()!=0 ){
306 string url = hrefUrl.front(); // 从队列的最开始取出一个网址
307 cout << url << endl;
308 BFS( url ); // 遍历提取出来的那个网页，找它里面的超链接网页放入hrefUrl，下载它里面的文本，图片
309 hrefUrl.pop(); // 遍历完之后，删除这个网址
310 }
311 WSACleanup();
312 return;
313 }
浙公网安备 33010602011771号