1 #ifndef _URL_H_030728_
2 #define _URL_H_030728_
3
4 #include <string>
5
6 const unsigned int URL_LEN = 256;
7 const unsigned int HOST_LEN = 256;
8
9 using namespace std;
10
11
12 enum url_scheme {
13 SCHEME_HTTP,
14 SCHEME_FTP,
15 SCHEME_INVALID
16 };
17
18 const int DEFAULT_HTTP_PORT = 80;
19 const int DEFAULT_FTP_PORT = 21;
20
21 class CUrl
22 {
23 public:
24 string m_sUrl; // 原始的url地址
25 enum url_scheme m_eScheme; // URL 类型
26
27 string m_sHost; // 提取出来的主机地址
28 int m_nPort; // 主机端口号
29 string m_sPath; //路径
30
31
32 public:
33 CUrl();
34 ~CUrl();
35
36 //bool ParseUrl(string strUrl);
37
38 // break an URL into scheme, host, port and request.
39 // result as member variants
40 bool ParseUrlEx(string strUrl);
41
42 // break an URL into scheme, host, port and request.
43 // result url as argvs
44 void ParseUrlEx(const char *url, char *protocol, int lprotocol,
45 char *host, int lhost,
46 char *request, int lrequest, int *port);
47
48 // get the ip address by host name
49 char *GetIpByHost(const char *host);
50
51 bool IsValidHost(const char *ip);
52 bool IsForeignHost(string host);
53 bool IsImageUrl(string url);
54 bool IsValidIp(const char *ip);
55 bool IsVisitedUrl(const char *url);
56 bool IsUnReachedUrl(const char *url);
57 bool IsValidHostChar(char ch);
58
59 //private:
60 void ParseScheme (const char *url);
61 };
62
63 extern pthread_mutex_t mutexMemory;
64
65 #endif /* _URL_H_030728_ */
1 /* URL handling
2 */
3
4 #include <iostream>
5 #include <string.h>
6 #include <sys/socket.h>
7 #include <netdb.h>
8 #include <map>
9 #include "Url.h"
10 #include <stdlib.h>
11 #include <arpa/inet.h>
12
13 //#include "Tse.h"
14 //#include "Url.h"
15 //#include "Http.h"
16 //#include "Md5.h"
17 //#include "StrFun.h"
18
19
20
21 //
22 ///* Is X "."? */
23 #define DOTP(x) ((*(x) == '.') && (!*(sdfx + 1)))
24 ///* Is X ".."? */
25 #define DDOTP(x) ((*(x) == '.') && (*(x + 1) == '.') && (!*(x + 2)))
26
27 map<string,string> mapCacheHostLookup;
28 //extern vector<string> vsUnreachHost;
29 //pthread_mutex_t mutexCacheHost = PTHREAD_MUTEX_INITIALIZER;
30 //extern set<string> setVisitedUrlMD5;
31 //extern map<unsigned long,unsigned long> mapIpBlock;
32 typedef map<string,string>::value_type valTypeCHL;
33
34 struct scheme_data
35 {
36 const char *leading_string;
37 int default_port;
38 int enabled;
39 };
40
41 /* 支持的网页类型 */
42 static struct scheme_data supported_schemes[] =
43 {
44 { "http://", DEFAULT_HTTP_PORT, 1 },
45 { "ftp://", DEFAULT_FTP_PORT, 1 },
46
47 /* 不合法的网页 */
48 { NULL, -1, 0 }
49 };
50
51 /* 分析类型，若是合法则返回正确的，否则是不合法的 */
52 void CUrl::ParseScheme (const char *url)
53 {
54 int i;
55
56 for (i = 0; supported_schemes[i].leading_string; i++)
57 if (0 == strncasecmp (url, supported_schemes[i].leading_string,strlen (supported_schemes[i].leading_string)))//比较url的前几个字母
58 {
59 if (supported_schemes[i].enabled)
60 {
61 this->m_eScheme = (enum url_scheme) i;
62 return;
63 }
64 else
65 {
66 this->m_eScheme = SCHEME_INVALID;
67 return;
68 }
69 }
70
71 this->m_eScheme = SCHEME_INVALID;
72 return;
73 }
74
75 /************************************************************************
76 * Function name: ParseUrlEx
77 * Input argv:
78 * -- strUrl: url
79 * Output argv:
80 * --
81 * Return:
82 true: success
83 false: fail
84 * Fucntion Description: break an URL into scheme, host, port and request.
85 * result as member variants
86 * Be careful: release the memory by the client
87 ************************************************************************/
88
89 bool CUrl::ParseUrlEx(string strUrl)
90 {
91 char protocol[10];
92 char host[HOST_LEN];
93 char request[256];
94 int port = -1;
95
96 memset( protocol, 0, sizeof(protocol) );
97 memset( host, 0, sizeof(host) );
98 memset( request, 0, sizeof(request) );
99
100 this->ParseScheme(strUrl.c_str());
101 if( this->m_eScheme != SCHEME_HTTP )
102 {
103 return false;
104 }
105
106 ParseUrlEx(strUrl.c_str(),
107 protocol, sizeof(protocol),
108 host, sizeof(host),
109 request, sizeof(request),
110 &port);
111
112 m_sUrl = strUrl;
113 m_sHost = host;
114 m_sPath = request;
115
116 if( port > 0 ){
117 m_nPort = port;
118 }
119
120 return true;
121 }
122
123 /************************************************************************
124 * Function name: ParseUrlEx
125 * Input argv:
126 * -- url: host name
127 * -- protocol: result protocol
128 * -- lprotocol: protocol length
129 * -- host: result host
130 * -- lhost: host length
131 * -- request: result request
132 * -- lrequest: request length
133 * Output argv:
134 * --
135 * Return:
136 true: success
137 false: fail
138 * Fucntion Description: break an URL into scheme, host, port and request.
139 * result as argvs
140 * Be careful:
141 ************************************************************************/
142 void CUrl::ParseUrlEx(const char *url,
143 char *protocol, int lprotocol,
144 char *host, int lhost,
145 char *request, int lrequest,
146 int *port)
147 {
148 char *work,*ptr,*ptr2;
149
150 *protocol = *host = *request = 0;
151 *port = 80;
152
153 int len = strlen(url);
154 //pthread_mutex_lock(&mutexMemory);
155 work = new char[len + 1];
156 //pthread_mutex_unlock(&mutexMemory);
157 memset(work, 0, len+1);
158 strncpy(work, url, len);
159 //把url的内容复制到work中
160
161 // find protocol if any
162 //在work中查找:(默认的是http)
163 ptr = strchr(work, ':');
164 if( ptr != NULL )
165 {
166 *(ptr++) = 0;
167 strncpy( protocol, work, lprotocol );
168 } else {
169 strncpy( protocol, "HTTP", lprotocol );
170 ptr = work;
171 }
172
173 // skip past opening /'s
174 //调过 //
175 if( (*ptr=='/') && (*(ptr+1)=='/') )
176 ptr+=2;
177
178 // 查找主机地址
179 ptr2 = ptr;
180 while( IsValidHostChar(*ptr2) && *ptr2 )
181 ptr2++;
182 *ptr2 = 0;//保证合法的字符串
183 strncpy( host, ptr, lhost );
184
185 //查找请求的网页
186 int offset = ptr2 - work;
187 const char *pStr = url + offset;
188 strncpy( request, pStr, lrequest );
189
190 //找到主机的端口
191 ptr = strchr( host, ':' );
192 if( ptr != NULL ){
193 *ptr = 0;
194 *port = atoi(ptr+1);
195 }
196
197 //pthread_mutex_lock(&mutexMemory);
198 delete [] work;
199 //pthread_mutex_unlock(&mutexMemory);
200 work = NULL;
201 }
202
203
204
205
206
207
208 /* scheme://user:pass@host[:port]...
209 * ^
210 * We attempt to break down the URL into the components path,
211 * params, query, and fragment. They are ordered like this:
212 * scheme://host[:port][/path][;params][?query][#fragment]
213 */
214
215 /*
216 bool CUrl::ParseUrl(string strUrl)
217 {
218 string::size_type idx;
219
220 this->ParseScheme(strUrl.c_str());
221 if( this->m_eScheme != SCHEME_HTTP )
222 return false;
223
224 // get host name
225 this->m_sHost = strUrl.substr(7);
226 idx = m_sHost.find('/');
227 if(idx != string::npos){
228 m_sHost = m_sHost.substr(0,idx);
229 }
230
231 this->m_sUrl = strUrl;
232
233 return true;
234 }
235 */
236 //CUrl的构造函数
237 CUrl::CUrl()
238 {
239 this->m_sUrl = "";
240 this->m_eScheme= SCHEME_INVALID;
241
242 this->m_sHost = "";
243 this->m_nPort = DEFAULT_HTTP_PORT; //默认端口
244
245 this->m_sPath = "";
246 /*
247 this->m_sParams = "";
248 this->m_sQuery = "";
249 this->m_sFragment = "";
250
251 this->m_sDir = "";
252 this->m_sFile = "";
253
254 this->m_sUser = "";
255 this->m_sPasswd = "";
256 */
257
258 }
259
260 CUrl::~CUrl()
261 {
262
263 }
264
265
266 /****************************************************************************
267 * Function name: GetIpByHost
268 * Input argv:
269 * -- host: host name
270 * Output argv:
271 * --
272 * Return:
273 ip: sucess
274 NULL: fail
275 * Function Description: get the ip address by host name
276 * Be careful: release the memory by the client
277 ****************************************************************************/
278 //通过主机地址获得IP地址
279 char * CUrl::GetIpByHost(const char *host)
280 {
281
282 if( !host ){ // null pointer
283 return NULL;
284 cout<<"f1";
285 }
286
287 if( !IsValidHost(host) ){ // invalid host
288 return NULL;
289 cout<<"f2";
290 }
291 unsigned long inaddr = 0;
292 char *result = NULL;
293 int len = 0;
294
295
296 inaddr = (unsigned long)inet_addr( host );//将字符串IP转化为32二进制的网络字节序
297 //if ( (int)inaddr != -1){
298 if ( inaddr != INADDR_NONE)
299 { // 主机地址就是用IP地址表示的
300 len = strlen(host);
301 //pthread_mutex_lock(&mutexMemory);
302 result = new char[len+1];
303 cout<<result;
304 //pthread_mutex_unlock(&mutexMemory);
305 memset(result, 0, len+1);
306 memcpy(result, host, len);
307
308 return result;
309 }
310 else
311 {
312 //firt find from cache
313
314 map<string,string>::iterator it = mapCacheHostLookup.find(host);
315 //可以在DNS缓存中找到
316 if( it != mapCacheHostLookup.end() )
317 { //如果在cache中找到IP地址
318 const char * strHostIp;
319
320 strHostIp = (*it).second.c_str();
321
322 inaddr = (unsigned long)inet_addr( strHostIp );
323 //if ( (int)inaddr != -1){
324 if ( inaddr != INADDR_NONE )
325 {
326 len = strlen(strHostIp);
327 //pthread_mutex_lock(&mutexMemory);
328 result = new char[len+1];
329 //pthread_mutex_unlock(&mutexMemory);
330 memset( result, 0, len+1 );
331 memcpy( result, strHostIp, len );
332
333 //cout << ":)" ;
334
335 return result;
336 }
337 }
338 }
339
340 //通过上面的方法我们都没有查找，这个时候我们只能通过DNS server查找了，这种带宽的消耗是必要的！
341 struct hostent *hp; /* Host entity */
342 hp = gethostbyname(host);
343 //通过主机号或者说是域名得到hostent结构，这个结构包含主机号或者说域名的很多信息，例如我们要找的IP字符串就在其中
344 if(hp == NULL) {
345 //cout << "gethostbyname() error in GetIpByHost: " << host << endl;
346 return NULL;
347 }
348
349 // cache host lookup
350 struct in_addr in;
351
352 bcopy(*(hp->h_addr_list), (caddr_t)&in, hp->h_length);
353 /*功能：将字符串src的前n个字节复制到dest中
354 说明：bcopy不检查字符串中的空字节NULL，函数没有返回值。*/
355
356 char abuf[INET_ADDRSTRLEN];
357 if( inet_ntop(AF_INET, (void *)&in,abuf, sizeof(abuf)) == NULL )
358 {
359 cout << "inet_ntop() return error in GetIpByHost" << endl;
360 return NULL;
361
362 }
363 else
364 {
365
366 //if( mapCacheHostLookup.count(host) == 0){
367 if( mapCacheHostLookup.find(host) == mapCacheHostLookup.end() ){
368
369 //cout << endl << host << " and " << abuf << endl;
370 mapCacheHostLookup.insert( valTypeCHL ( host, abuf));
371 //更新DNS缓存
372 //cout<<((*mapCacheHostLookup.find("home.ustc.edu.cn")).second.c_str());
373
374 }
375
376 }
377
378 // return result
379 len = strlen(abuf);
380 //pthread_mutex_lock(&mutexMemory);
381 result = new char[len + 1];
382 //pthread_mutex_unlock(&mutexMemory);
383 memset( result, 0, len+1 );
384 memcpy( result, abuf, len );
385
386 return result;
387 }
388
389 /**********************************************************************************
390 * Function name: IsValidHostChar
391 * Input argv:
392 * -- ch: the character for testing
393 * Output argv:
394 * --
395 * Return:
396 true: is valid
397 false: is invalid
398 * Function Description: test the specified character valid
399 * for a host name, i.e. A-Z or 0-9 or -.:
400 **********************************************************************************/
401 bool CUrl::IsValidHostChar(char ch)
402 {
403 return( isalpha(ch) || isdigit(ch)
404 || ch=='-' || ch=='.' || ch==':' || ch=='_');
405 }
406
407 /**********************************************************************************
408 * Function name: IsValidHost
409 * Input argv:
410 * -- ch: the character for testing
411 * Output argv:
412 * --
413 * Return:
414 true: is valid
415 false: is invalid
416 * Function Description: test the specified character valid
417 * for a host name, i.e. A-Z or 0-9 or -.:
418 * Be careful:
419 **********************************************************************************/
420 bool CUrl::IsValidHost(const char *host)
421 {
422 if( !host ){//空的主机号，我们认为是无效的主机号
423 return false;
424 }
425
426 if( strlen(host) < 6 ){ //主机号长度小于6，我们认为ieshi无效的主机号
427 return false;
428 }
429
430 char ch;
431 for(unsigned int i=0; i<strlen(host); i++){
432 ch = *(host++);
433 if( !IsValidHostChar(ch) ){
434 return false;
435 }
436 }
437
438 return true;
439 }
440
441 /**********************************************************************************
442 * Function name: IsVisitedUrl
443 * Input argv:
444 * -- url: url
445 * Output argv:
446 * --
447 * Return:
448 true: is visited
449 false: not visited
450 * Function Description: test the url visited by the MD5
451 * Be careful:
452 **********************************************************************************/
453 bool CUrl::IsVisitedUrl(const char *url)//判断该URL是否访问过
454 {
455 if( !url ){
456 return true; // if be null, we think it have been visited
457 }
458
459 CMD5 iMD5;
460 iMD5.GenerateMD5( (unsigned char*)url, strlen(url) );
461 string strDigest = iMD5.ToString();
462
463 if( setVisitedUrlMD5.find(strDigest) != setVisitedUrlMD5.end() ) {
464 return true;
465 } else {
466 return false;
467 }
468
469 }
470
471
472 /**********************************************************************************
473 * Function name: IsValidIp
474 * Input argv:
475 * -- ip: ip
476 * Output argv:
477 * --
478 * Return:
479 true: inside the ip block
480 false: outside the ip block
481 * Function Description: decide teh ip whether or not inside the ip block
482 * Be careful:
483 **********************************************************************************/
484 bool CUrl::IsValidIp(const char *ip)
485 {
486 if( ip == NULL )
487 {
488 return false;
489 }
490
491 unsigned long inaddr = (unsigned long)inet_addr(ip);
492 if( inaddr == INADDR_NONE ){//显然该IP参数不是正确的字符串IP
493 return false;
494 }
495
496 if (mapIpBlock.size() > 0) { //判断是否要过滤掉
497 map<unsigned long, unsigned long>::iterator pos;
498 for (pos = mapIpBlock.begin(); pos != mapIpBlock.end(); ++pos) {
499 unsigned long ret;
500
501 ret = inaddr & ~((*pos).second);
502 if (ret == (*pos).first) { // inside
503 return true;
504 }
505 }
506
507 // outside
508 return false;
509 }
510
511
512 // if block range is not given, we think it inside also
513 return true;
514 }
515 /*
516 * If it is, return true; otherwise false
517 * not very precise
518 */
519 bool CUrl::IsForeignHost(string host)
520 {
521 if( host.empty() ) return true;
522 if( host.size() > HOST_LEN ) return true;
523
524 unsigned long inaddr = 0;
525
526 inaddr = (unsigned long)inet_addr( host.c_str() );
527 if ( inaddr != INADDR_NONE){ // host is just ip
528 return false;
529 }
530
531 string::size_type idx = host.rfind('.');
532 string tmp;
533 if( idx != string::npos ){
534 tmp = host.substr(idx+1);
535 }
536
537 CStrFun::Str2Lower( tmp, tmp.size() );
538 const char *home_host[] ={
539 "cn","com","net","org","info",
540 "biz","tv","cc", "hk", "tw"
541 };
542
543 int home_host_num = 10;
544
545 for(int i=0; i<home_host_num; i++){
546 if( tmp == home_host[i] )
547 return false;
548 }
549
550 return true;
551 }
552
553
554 bool CUrl::IsImageUrl(string url)
555 {
556 if( url.empty() ) return false;
557 if( url.size() > HOST_LEN ) return false;
558
559 string::size_type idx = url.rfind('.');
560 string tmp;
561 if( idx != string::npos ){
562 tmp = url.substr(idx+1);
563 }
564
565 CStrFun::Str2Lower( tmp, tmp.size() );
566 const char *image_type[] ={
567 "gif","jpg","jpeg","png","bmp",
568 "tif","psd"
569 };
570
571 int image_type_num = 7;
572
573 for (int i=0; i<image_type_num; i++)
574 {
575 if( tmp == image_type[i] )
576 return true;
577 }
578
579 return false;
580 }
浙公网安备 33010602011771号