php利用curl抓取

<?php
set_time_limit(0);




function getUrl() {
$data = array (


//新闻阅读频道采集地址

 'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3366&region=&page='=>array('page'=>3,'cate'=>'新闻阅读'),

 'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3376&region=&page='=>array('page'=>1,'cate'=>'新闻阅读'),
  'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3371&region=&page='=>array('page'=>2,'cate'=>'新闻阅读'),
 
//综艺明星
  'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3396&region=&page='=>array('page'=>5,'cate'=>'综艺明星'),
 
  //科技数码频道采集地址
 'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3369&region=&page='=>array('page'=>2,'cate'=>'科技数码'),

//生活购物
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3392&region=&page='=>array('page'=>3,'cate'=>'生活购物'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3476&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3382&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3385&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3426&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3393&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
//影音娱乐
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3432&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3397&region=&page='=>array('page'=>1,'cate'=>'影音娱乐'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3406&region=&page='=>array('page'=>1,'cate'=>'影音娱乐'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3432&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3398&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3386&region=&page='=>array('page'=>1,'cate'=>'影音娱乐'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3395&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3398&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3404&region=&page='=>array('page'=>1,'cate'=>'影音娱乐'),
//社区交友
//文化教育频道采集地址
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3372&region=&page='=>array('page'=>1,'cate'=>'文化教育'),
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3378&region=&page='=>array('page'=>1,'cate'=>'文化教育'),
//地方政务
//公共名人
'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3413&region=&page='=>array('page'=>1,'cate'=>'公共名人'),



/*
 *
 *http://wei.daxues.cn
 */
//新闻阅读
//'http://wei.daxues.cn/index.php?m=weixin&a=index&id=1&p='=>array('p'=>2,'cate'=>'新闻阅读'),
//综艺明星
//'http://wei.daxues.cn/index.php?m=weixin&a=index&id=44&p='=>array('p'=>15,'cate'=>'综艺明星'),
//生活购物
//'http://wei.daxues.cn/index.php?m=weixin&a=index&id=47&p='=>array('p'=>99,'cate'=>'生活购物'),
//影音娱乐
//'http://wei.daxues.cn/index.php?m=weixin&a=index&id=48&p='=>array('p'=>54,'cate'=>'影音娱乐'),
//社区交友
//'http://wei.daxues.cn/index.php?m=weixin&a=index&id=49&p='=>array('p'=>21,'cate'=>'社区交友'),
//文化教育频道采集地址
//'http://wei.daxues.cn/index.php?m=weixin&a=index&id=26&p='=>array('p'=>26,'cate'=>'文化教育'),
);

foreach ($data as $urlx => $pagex ) {
        $pages = range ( 1, $data[$urlx]['page'] );
        
        foreach ( $pages as $p ) {
            $xx = $urlx . $p;
            $content = get ( $xx );
            $content = str_replace ( "\r\n", '', $content );
            $content = str_replace ( "\n", '', $content );
            $content = str_replace ( "\t", '', $content );
            preg_match_all ( '/(http:\/\/www.vx173.com\/content-\d+-\d+-\d+.html)/', $content, $m );
            $m = isset ( $m[1]  ) ? $m[1]  : array ();
            $arrUrl=array_unique($m);
            //$arrrUrl+=$arrUrl;
           
            $arrCate= $data[$urlx]['cate'];   
            $con=mysql_connect("127.0.0.1","root","");
            if(!$con){
            die("连接失败");
            }
            mysql_set_charset('utf8');
            mysql_select_db("caiji",$con);
            foreach($arrUrl as $val){
            // $val='http://wei.daxues.cn/'.$val;
           //  $sql="select * from urls";
             $sql="insert into urls (cate,url) values ('$arrCate','$val');";
             $cn=mysql_query($sql);
          
           
            //echo "<pre>";
           // print_r($arrUrl[0]);
            //}

  }
}
    
//insert db
}
}
function getDetail($url = '') {

    $data=getData();
    echo "<pre>";
    print_r($data);
    exit;
 

    foreach ($data as $urlx => $pagex ) {
        $pages = range ( 1, $data[$urlx]['p'] );
        
        foreach ( $pages as $p ) {
            $xx = $urlx . $p;
            $content = get ( $xx );
            $content = str_replace ( "\r\n", '', $content );
            $content = str_replace ( "\n", '', $content );
            $content = str_replace ( "\t", '', $content );
            preg_match_all ( '/(http:\/\/www.vx173.com\/content-\d+-\d+-\d+.html)/', $content, $m );
            $m = isset ( $m[1]  ) ? $m[1]  : array ();
            $arrUrl=array_unique($m);
            //$arrrUrl+=$arrUrl;
          // print_r($arrUrl);
            $arrCate= $data[$urlx]['cate'];   

            foreach($arrUrl as $val){
    $content = get ( $val );
    $content = str_replace ( "\r\n", '', $content );
    $content = str_replace ( "\n", '', $content );
    $content = str_replace ( "\t", '', $content );
    
    #preg_match_all('/Hao1">k.*?<img src="(.*?)">/',$content,$l);

    preg_match_all('/<img.*?src="(.*?)".*?width="225" height="225"/',$content,$l);
 
    preg_match_all ( '/官方网站：.*?>(.*?)</',$content, $m );
    preg_match_all ( '/原始账号：.*?>(.*?)/', $content, $n );
    preg_match_all ( '/所属分类：.*?>(.*?)/', $content, $e );
    preg_match_all ( '/所属地区：.*?>(.*?)/', $content, $f );
    preg_match_all ( '/官方微博：.*?>(.*?)</',$content, $g );
    preg_match_all ( '/收录时间：\d+-\d+-\d+/',$content, $h );
    preg_match_all ( '/公众账号：.*?>(.*?)</',$content, $k );
    
    preg_match_all ( '/style="font-size:14px; line-height: 30px(.*?)(.*?)<\/div/', $content, $j );


    /*
    echo '<pre>';
    print_r ( $m[0] );
    print_r ( $n[0]);
    print_r ( $e[0]);
    print_r ( $f[0]);
    print_r ( $g[0]);
    print_r ( $h[0]);
    print_r ( $l[0]);
    exit;
   */
    $arr=array_merge($m[0],$n[0],$e[0],$f[0],$g[0],$h[0],$j[0],$l[1],$k[0]);
    echo "<pre>";
    print_r($arr);

    exit;

    @$arrr=str_replace( 'style="font-size:14px; line-height: 30px;"> ', "", $arr[6]);
    $arr[6]=str_replace( '<', "", $arrr);
     
     $xulie=serialize($arr);
    //数据过滤
    $con=mysql_connect("127.0.0.1","root","");
            if(!$con){
            die("连接失败");
            }
            mysql_set_charset('utf8');
            mysql_select_db("caiji",$con);
            //foreach($arr as $val){
             $sql="insert into data (data,fk_cate) values ('$xulie','$arrCate');";
             $cn=mysql_query($sql);

          //  }

 }
   


  }
    }


// exit;


}

function get($url) {
    // 初始化，返回一个handler
    $ch = curl_init ( $url );
    // 设置选项，有返回值
    curl_setopt ( $ch, CURLOPT_RETURNTRANSFER, 1 );
    // 设置选项，来源页，这意味着可以伪造referer达到某种目的
    curl_setopt ( $ch, CURLOPT_REFERER, 'http://www.baidu.com/' );
    // 设置选项，浏览器信息
    curl_setopt ( $ch, CURLOPT_USERAGENT, 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322)' );
    // 执行
    $raw = curl_exec ( $ch );
    curl_close ( $ch ); // 关闭handler
    // 输出结果
    return $raw;
}

$result = getUrl ();
//$result = getDetail ();

echo '<pre>';
print_r ( $result );