随笔-68  评论-89  文章-0 

C#实现web信息抓取

        随着Internet的普及,网络信息正以极高的速度增长,在这么多数据中找到自己需要的信息是一件很繁琐的事情,找到需要的信息后如何获取也是件麻烦的事。这就需要Internet信息抓取程序来代替人工的操作。
            所谓Internet信息抓取程序,就是程序会按照用户的关键词或关键网站来收集相应的信息,并提供给用户想要的信息格式。
            信息量的增加会带来信息网站发布人员工作量的剧增,为实现信息发布系统实现信息自
            动发布、减少工作人员工作量、即时跟踪最新信息,就需要自动信息提供程序,因此Internet信息抓取程序应运而生。
            
            目标

            实现自定义网站信息分类抓取,存入本地数据库、生成静态页面或其它用户定义的信息结构,并下载与信息相关的多媒体文件。
            
            开发
            
            l         目标站点结构分析
            本步骤是准确抓取信息个关键。
            首先要选择更新频率高的页面做为抓取地址,然后分析要抓取内容页面url特点。
            然后分析要抓取信息页面的元素特性,比如标题位置,内容位置 等,得到定位标记点。
            将以上信息写成自己的配置文件或存到数据库中。
            每个网站都需要分析,写出单独的配置文件,供抓取程序使用。
            
            l         信息提取
            根据配置文件取得要抓取页面url,使用HttpWebRequest类获取内容:

            //获取http页面函数

                    public string Get_Http(string a_strUrl,int timeout)

                    {

                        string strResult ;        

            

                        try

                        {

            HttpWebRequest myReq =
            (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;

                            myReq.Timeout = timeout;

                            HttpWebResponse HttpWResp =
            (HttpWebResponse)myReq.GetResponse();

                       

                            Stream myStream = HttpWResp.GetResponseStream () ;

            

                            StreamReader sr = new StreamReader(myStream ,
            Encoding.Default);

                            StringBuilder strBuilder = new StringBuilder();

                            while (-1 != sr.Peek())

                            {

                                strBuilder.Append(sr.ReadLine()+"\r\n");

                            }

            

                            strResult = strBuilder.ToString();

                        }

                        catch(Exception exp)

                        {

                            strResult = "错误:" + exp.Message ;

                        }

            

                        return strResult ;

            

                    }

            获取页面内容后,分析页面中连接地址取到要抓取的url:

            //处理页面标题和链接

                    public string SniffWebUrl( string urlStr,string
            blockB,string blockE )

                    {      

                        string urlch1 = "";

                        string urlch2 = "";                   

                        int end_n1 = 0;

                        int end_nums = 0;

                        int end_nums1 = 0;

                        int end_nums2 = 0;

                        int end_nums3     = 0;           

                        string reUTStr = "";

                        string reTitle = "";

                        string ret = "";          

                        try

                        {

                            int pos01 = urlStr.IndexOf( "." );

                            int pos02 = urlStr.LastIndexOf( "/" );

                            if( pos01 < 0 )

                            {

                                return "";

                            }

                            if( pos02 < 0 )

                            {

                                return "";

                            }

                            int pos03 = urlStr.IndexOf( "/",pos01 );

                            if ( pos03 < 0 )

                            {

                                urlch1 = urlStr;

                                urlch2 = urlStr;

                            }

                            else

                            {

                                urlch1 = urlStr.Substring( 0,pos03 );

                                urlch2 = urlStr.Substring( 0,pos02 );

                            }

            

                            string tmpAllStr = new PublicFun().Get_Http( urlStr
            ,time1);

            

                            int pos1 = tmpAllStr.IndexOf( blockB );

                            int pos2 = tmpAllStr.IndexOf( blockE,pos1 +
            blockB.Length );

                            if ( pos1>0 && pos2>0 && pos2>pos1 )

                            {

                                ret = tmpAllStr.Substring( pos1 +
            blockB.Length,pos2 - pos1 - blockB.Length );

                                ret = ret.Substring( ret.IndexOf( "<" ));

                                while( ret.IndexOf( "<A" ) >= 0 )

                                {

                                    ret = ret.Substring( 0,ret.IndexOf( "<A" ) )
            + "<a" + ret.Substring( ret.IndexOf( "<A" ) + 2 );

                                }

                                while( ret.IndexOf( "</A" ) >=0 )

                                {

                                    ret = ret.Substring( 0,ret.IndexOf( "</A" )
            ) + "</a" + ret.Substring( ret.IndexOf( "</A" ) + 3 );

                                }

                                while( ret.IndexOf( "Href=" ) >=0 )

                                {

                                    ret = ret.Substring( 0,ret.IndexOf( "Href="
            )) + "href=" + ret.Substring( ret.IndexOf( "Href=" ) + 5 );

                                }

                                while( ret.IndexOf( "HREF=" ) >=0 )

                                {

                                    ret = ret.Substring( 0,ret.IndexOf( "HREF="
            )) + "href=" + ret.Substring( ret.IndexOf( "HREF=" ) + 5 );

                                }

                                while( ret.IndexOf( "href='" ) >=0 )

                                {

                                    ret = ret.Substring( 0,ret.IndexOf( "href='"
            )) + "href=\"" + ret.Substring( ret.IndexOf( "href='" ) + 6 );

                                }

                            }      

                            tmpAllStr = ret;     

                            int begin_nums = tmpAllStr.IndexOf( "href=" );

            

                            while ( begin_nums >= 0 )

                            {              

                                string tmpStrA = "";

                                string tmpStrB = tmpAllStr.Substring( begin_nums
            + 5,1 );

                                if ( tmpStrB == "\"" )

                                {

                                    end_n1 = begin_nums + 6;

                                    if ( ( end_n1 + 1 ) > tmpAllStr.Length )

                                    {

                                        return "";

                                    }

                                    tmpStrA = tmpAllStr.Substring(
            begin_nums+6,1 );

                                }

                                else

                                {

                                    end_n1 = begin_nums + 5;

                                    tmpStrA = tmpStrB;

                                }

            

                                if ( tmpStrA == "#" )

                                {

                                    tmpAllStr = tmpAllStr.Substring( end_n1 );

                                    begin_nums = tmpAllStr.IndexOf( "href=" );

                                }

                                else

                                {                  

                                    end_nums1 = tmpAllStr.IndexOf( " ",end_n1 );

                                    end_nums2 = tmpAllStr.IndexOf( ">",end_n1 );

                                    end_nums3 = tmpAllStr.IndexOf(
            "</a",end_nums2 );

            

                                    if ( ( end_nums3 >= 0 ) && ( end_nums2 >= 0
            ) )

                                    {

                                        reTitle = tmpAllStr.Substring( end_nums2
            + 1,end_nums3 - end_nums2 - 1 );

            

                                        if ( end_nums1 > end_nums2 )

                                        {

                                            end_nums = end_nums2;

                                        }

                                        else

                                        {

                                            if ( end_nums1 < 0 )

                                            {

                                                end_nums = end_nums2;

                                            }

                                            else

                                            {

                                                end_nums = end_nums1;

                                            }

                                        }

                                        string str4 = tmpAllStr.Substring(
            end_nums-1, end_nums - end_nums + 1 );

            

                                        if ( str4 =="\""  || str4 == "'" )

                                        {

                                            end_nums = end_nums - 1;

                                        }

                                        string sTotalOne = tmpAllStr.Substring(
            end_n1,end_nums - end_n1 );

            

                                        if ( sTotalOne.IndexOf( "http://" ) <0 )

                                        {

                                            if ( sTotalOne.IndexOf( "/" ) == 0 )

                                            {

                                                sTotalOne = urlch1 + sTotalOne;

                                            }

                                            else

                                            {                              

                                                int linshiIntNum = 0;

                                                int flags = 0;

                                                string urlChange = urlStr;;

                                                while( sTotalOne.IndexOf( "../"
            ) >= 0 )

                                                {

                                                    sTotalOne =
            sTotalOne.Substring( sTotalOne.IndexOf( "../" ) + 3 );

                                                    linshiIntNum = linshiIntNum
            + 1;

                                                    flags = flags +1;

                                                }

                                                while( ( urlChange.LastIndexOf(
            "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )

                                                {

                                                    urlChange =
            urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );

                                                    linshiIntNum = linshiIntNum
            - 1;

                                                }

                                                if ( flags == 0 )

                                                {

                                                    sTotalOne = urlch2 + "/" +
            sTotalOne;

                                                }

                                                else

                                                {

                                                    sTotalOne = urlChange + "/"
            + sTotalOne;

                                                }

                                            }

                                        }

                                        reUTStr = reUTStr + new
            PublicFun().RemoveHtmlCode( reTitle ) + sTotalOne;

            

                                        tmpAllStr = tmpAllStr.Substring(
            end_nums3 + 4 );

                                        begin_nums = tmpAllStr.IndexOf( "href="
            );

                                    }

                                    else

                                    {

                                        begin_nums = -1;

                                    }                   

                                }

                            }

                            return reUTStr;

                        }

                        catch( Exception e)

                        {

                            return "";

                        }

                    }

            

            得到要抓取内容的url后,处理该页面:

            //获取链接内容并分类处理

                    public string GetWebContent( string gatherUrl,string
            subUrl,string subTitle,string b_Content,string e_Content,string
            b_Filter,string e_Filter,string root )

                    {

                        string tmpAllStr = "";           

                        string dfStrB = "";

                        string dfStrE = "";               

                        string rePicStr = "";//图片返回路径   

                        string reContentStr = "";

                        string picHtml = "images"; //本地图片路径

                       

                        string urlch1 ="";

                        string urlch2 ="";

                        int pos1 = gatherUrl.IndexOf( "." );

                        int pos2 = gatherUrl.LastIndexOf( "/" );

                        if( pos1 < 0 )

                        {

                            return "";

                        }

                        if( pos2 < 0 )

                        {               

                            return "";

                        }

                        int pos3 = gatherUrl.IndexOf( "/",pos1 );

                        if ( pos3 < 0 )

                        {

                            urlch1 = gatherUrl;

                            urlch2 = gatherUrl;

                        }

                        else

                        {

                            urlch1 = gatherUrl.Substring( 0,pos3 );

                            urlch2 = gatherUrl.Substring( 0,pos2 );

                        }   

                       

                        tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );

                        //取稿源

                        string docFromStr = "";

                        if ( dfStrB != "" && dfStrE != "" )

                        {

                            if ( tmpAllStr != "" )

                            {

                                int b_docF = tmpAllStr.IndexOf( dfStrB );

                                if ( b_docF > 0 )

                                {

                                    int e_docF = tmpAllStr.IndexOf(
            dfStrE,b_docF + dfStrB.Length );

                                    if ( e_docF > 0 && e_docF > b_docF && e_docF
            - b_docF < 20 )

                                    {

                                        docFromStr = tmpAllStr.Substring( b_docF
            + dfStrB.Length, e_docF - b_docF - dfStrB.Length );

                                    }

                                }

                            }

                        }

                        //取内容

                        if ( tmpAllStr != "" )

                        {               

                            int begin_strnum = tmpAllStr.IndexOf( b_Content );

                            if ( begin_strnum < 0 )

                            {                  

                                return "";

                            }

                            int end_strnum = tmpAllStr.IndexOf(
            e_Content,begin_strnum + b_Content.Length );

                            if ( end_strnum < 0 )

                            {                  

                                return "";

                            }

                            string sTotalSubM = "";

                            if ( end_strnum > begin_strnum )

                            {

                                sTotalSubM = tmpAllStr.Substring (
            begin_strnum,end_strnum - begin_strnum );

                            }

                           

                            if ( sTotalSubM == "" )

                            {                  

                                return "";

                            }              

                            //过滤无用信息

                            int bfnum = sTotalSubM.IndexOf( b_Filter );

                            if ( bfnum > -1 )

                            {

                                int efnum = sTotalSubM.IndexOf( e_Filter,bfnum
);

                                if ( efnum > -1 )

                                {

                                    if ( efnum > bfnum )

                                    {

                                        sTotalSubM = sTotalSubM.Substring(
            0,bfnum ) + sTotalSubM.Substring( efnum + e_Filter.Length );

                                    }

                                }

                            }

                            //格式化图片标记

                           

                            while( sTotalSubM.IndexOf( "Src=" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "Src=" ) ) + "src=" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "Src=" ) + 4 );

                            }

                            while( sTotalSubM.IndexOf( "SRC=" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "SRC=" ) ) + "src=" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "SRC=" ) + 4 );

                            }

                            while( sTotalSubM.IndexOf( "src='" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "src='" ) ) + "src=\"" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "src='" ) + 5 );

                            }

            

                            //取图片地址

                            int end_n12 = 0;

                            int end_nums2 = 0;

                            int begin_nums2 = sTotalSubM.IndexOf( "src=" );

                            while( begin_nums2 >= 0 )

                            {

                                String tmpStr = sTotalSubM.Substring(
            begin_nums2 + 4,1 );

                                if ( tmpStr == "\"" )

                                {

                                    end_n12 = begin_nums2 + 5;

                                }

                                else

                                {

                                    end_n12 = begin_nums2 + 4;

                                }

                                int end_nums2a = sTotalSubM.IndexOf( " ",end_n12
            );

                                int end_nums2b = sTotalSubM.IndexOf( ">",end_n12
            );

                                if ( end_nums2b < 0 )

                                {

                                    break;

                                }

                                if ( end_nums2a > end_nums2b )

                                {

                                    end_nums2 = end_nums2b;

                                }

                                else

                                {

                                    if (end_nums2a<0)

                                    {

                                        end_nums2 = end_nums2b;

                                    }

                                    else

                                    {

                                        end_nums2 = end_nums2a;

                                    }

                                }

                                tmpStr = sTotalSubM.Substring( end_nums2-1,1 );

                                if ( tmpStr == "\"" || tmpStr == "'" )

                                {

                                    end_nums2 = end_nums2 - 1;

                                }

                                string tmpPicStr = sTotalSubM.Substring(
            end_n12,end_nums2 - end_n12 );

            

                                if ( tmpPicStr.IndexOf( "http://" ) < 0 )

                                {

                                    if ( tmpPicStr.IndexOf( "/" ) == 0 )

                                    {

                                        tmpPicStr = urlch1 + tmpPicStr;

                                    }

                                    else

                                    {                           

                                        int linshiIntNum = 0;

                                        int flags = 0;

                                        string urlChange = subUrl;

                                        while( tmpPicStr.IndexOf( "../" ) >= 0 )

                                        {

                                            tmpPicStr = tmpPicStr.Substring(
            tmpPicStr.IndexOf("../") + 3 );

                                            linshiIntNum = linshiIntNum + 1;

                                            flags = flags + 1;

                                        }

                                        while( ( urlChange.LastIndexOf( "/" ) >=
            0 ) && ( linshiIntNum >= 0 ) )

                                        {

                                            urlChange = urlChange.Substring(
            0,urlChange.LastIndexOf( "/" ) );

                                            linshiIntNum = linshiIntNum - 1;

                                        }

                                        if ( flags == 0 )

                                        {

                                            tmpPicStr = urlch2 + "/" +
tmpPicStr;

                                        }

                                        else

                                        {

                                            tmpPicStr = urlChange + "/" +
            tmpPicStr;

                                        }

                                    }

                                }

                                //tmpPicStr = tmpPicStr.ToLower();

                                string tmpPicStrTmp = tmpPicStr.ToLower();

                                //if ( tmpPicStr.IndexOf( ".jpg" ) > 0 ||
            tmpPicStr.IndexOf( ".gif" ) > 0 || tmpPicStr.IndexOf( ".bmp" ) > 0 )

                                if ( tmpPicStrTmp.IndexOf( ".jpg" ) > 0 ||
            tmpPicStrTmp.IndexOf( ".gif" ) > 0 || tmpPicStrTmp.IndexOf( ".bmp" )
            > 0 )

                                {

                                    rePicStr = rePicStr + "||" + tmpPicStr ;

            

                                    int flagN2 = tmpPicStr.LastIndexOf( "/" );

                                    string fileN2 = picHtml +
            tmpPicStr.Substring( flagN2 );

                                    sTotalSubM = sTotalSubM.Substring(
            0,end_nums2 ) + ">******" + fileN2 + "******<" +
            sTotalSubM.Substring( end_nums2 );

            

                                    begin_nums2 = sTotalSubM.IndexOf( "src=",
            end_nums2 + fileN2.Length + 22 );

                                }

                                else

                                {

                                    begin_nums2 = sTotalSubM.IndexOf( "src=",
            end_nums2 + 4 );                       

                                }                  

                            }

                            if ( rePicStr.Length > 2 ) 

                                rePicStr =  rePicStr.Substring(2);              


            

                            //内容处理 格式化关键标记

                            while( sTotalSubM.IndexOf( "<P" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "<P" ) ) + "|****|<" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "<P" ) + 2 );

                            }

                            while( sTotalSubM.IndexOf( "<p" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "<p" ) ) + "|****|<" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "<p" ) + 2 );

                            }

                            while( sTotalSubM.IndexOf( "</P" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "</P" ) ) + "|****|<" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "</P" ) + 3 );

                            }

                            while( sTotalSubM.IndexOf( "</p" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "</p" ) ) + "|****|<" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "</p" ) + 3 );

                            }

                            while( sTotalSubM.IndexOf( "<br" ) >=0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "<br" ) ) + "+****+<" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "<br" ) + 3 );

                            }

                            while( sTotalSubM.IndexOf( "<BR" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "<BR" ) ) + "+****+<" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "<BR" ) + 3 );

                            }

                            while( sTotalSubM.IndexOf( "<Br" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "<Br" ) ) + "+****+<" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "<Br" ) + 3 );

                            }

                            while( sTotalSubM.IndexOf( "<bR" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "<bR" ) ) + "+****+<" + sTotalSubM.Substring(
            sTotalSubM.IndexOf( "<bR" ) + 3 );

                            }

            

                            //去除html标记

                            int linshiInt1 = sTotalSubM.IndexOf( "<" );

                            int linshiInt2 = sTotalSubM.IndexOf( ">" );         
             

            

                            if ( linshiInt2 < linshiInt1 )

                            {

                                sTotalSubM = sTotalSubM.Substring( linshiInt2 +
            1 );

                            }

                            int linshiInt11 = sTotalSubM.LastIndexOf( "<" );

                            int linshiInt12 = sTotalSubM.LastIndexOf( ">" );

                            if ( linshiInt12 < linshiInt11 )

                            {

                                sTotalSubM = sTotalSubM.Substring( 0,linshiInt12
            + 1 );

                            }

                            linshiInt1 = sTotalSubM.IndexOf( "<" );

                            while ( linshiInt1 >= 0 )

                            {

                                linshiInt2 = sTotalSubM.IndexOf( ">",linshiInt1
            );

                                if ( linshiInt2 >= 0 )

                                {              

                                    sTotalSubM = sTotalSubM.Substring(
            0,linshiInt1 ) + sTotalSubM.Substring( linshiInt2 + 1 );

                                }

                                else

                                {

                                    sTotalSubM = sTotalSubM.Substring(
            0,linshiInt1 );

                                }

                                linshiInt1 = sTotalSubM.IndexOf("<");

                            }

            

                            //还原关键标记

                            int linshiInt3 = 0;

                            int linshiInt4 = 0;

            

                            while( sTotalSubM.IndexOf( "+****+" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "+****+" ) ) + "<br>\n" +
            sTotalSubM.Substring( sTotalSubM.IndexOf( "+****+" ) + 9 );

                            }

                            while( sTotalSubM.IndexOf( "|****|" ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( "|****|" ) ) + "<br>\n" +
            sTotalSubM.Substring( sTotalSubM.IndexOf( "|****|" ) + 9 );

                            }

                            while( sTotalSubM.IndexOf( "******" ) >= 0 )

                            {

                                linshiInt3 = sTotalSubM.IndexOf( "******" ) + 9;

                                linshiInt4 = sTotalSubM.IndexOf(
            "******",linshiInt3 );

                                if ( linshiInt4 >= 0 )

                                {

                                    int tmpPos = sTotalSubM.IndexOf( "******" );

                                    string tmpStr1 = sTotalSubM.Substring(
            0,tmpPos );

                                    string tmpStr2 = sTotalSubM.Substring(
            linshiInt3,linshiInt4 - linshiInt3 );

                                    string tmpStr3 = sTotalSubM.Substring(
            linshiInt4 + 9 );

                                    sTotalSubM = tmpStr1 + "<img src=" + tmpStr2
            + ">" + tmpStr3;

                                }

                                else

                                {

                                    break;

                                }

                            }

                            //去除内容中的标题

                            if ( sTotalSubM.IndexOf( subTitle ) >= 0 )

                            {

                                sTotalSubM = sTotalSubM.Substring(
            0,sTotalSubM.IndexOf( subTitle ) ) + sTotalSubM.Substring(
            sTotalSubM.IndexOf( subTitle ) + subTitle.Length );

                            }

                            reContentStr = sTotalSubM;

                            //调用下载图片功能

                            //下载图片到指定目录

                            string[] img_Url = new PublicFun().split(
            rePicStr,"||" );

                            for ( int i=0;i<img_Url.Length;i++ )

                            {

                                if ( img_Url[i] != "" )

                                {

                                    new PublicFun().Get_Img(
            img_Url[i],10000,root + "\\images\\" + img_Url[i].Substring(
            img_Url[i].LastIndexOf("/")+1 ) );

                                }

                            }

                        }

                        return reContentStr;

                    }

            以上方法返回要取得的信息,包括标题内容,图片地址等。

            

            下载页面中图片:

            //下载图片

                    public void Get_Img(string a_strUrl,int timeout,string
            filepath)

                    {

                        try

                        {

            HttpWebRequest myReq =
            (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;

                            myReq.Timeout = timeout;

                            HttpWebResponse HttpWResp =
            (HttpWebResponse)myReq.GetResponse();        

                            Stream myStream = HttpWResp.GetResponseStream () ;  
                  

                            Bitmap map = new Bitmap( myStream );

                            PictureBox picB = new PictureBox();

                            picB.Image = (Image)map;

                            string path = filepath.Substring(
            0,filepath.LastIndexOf( "\\" ) );

                            if (!Directory.Exists(path))

                            {

                                CreateDir( path );

                            }              

                            picB.Image.Save(filepath);               

                        }

                        catch(Exception exp)

                        {

                            string ss = exp.Message;

                    WriteLog( filepath.Substring(0,filepath.LastIndexOf("\\")) +
            "\\error.log",a_strUrl + "--" + ss + "\r\n");       

                        }

                    }


            l         保存文件或入库
            上面取得的信息可以按自己的要求保存。
            
            ****设计的时候没有使用url按层次循环抓取,这样定义抓取url效率更高,速度更快。
            
            
            测试程序下载:http://bjfile.focus.cn/file/15379/NetBugV102.rar

posted on 2007-09-18 23:48 噢耶游戏 阅读(...) 评论(...) 编辑 收藏