数据抓取

<?php
set_time_limit ( 0 );
function getData() {
    $data = array (//新闻阅读频道采集地址
/*
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3366&region=&page=' => array ('page' => 3, 'cate' => '新闻阅读' ),


     'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3376&region=&page='=>array('page'=>1,'cate'=>'新闻阅读'),
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3371&region=&page='=>array('page'=>2,'cate'=>'新闻阅读'),
    

    //综艺明星
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3396&region=&page='=>array('page'=>5,'cate'=>'综艺明星'),
    

    //科技数码频道采集地址
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3369&region=&page='=>array('page'=>2,'cate'=>'科技数码'),
    

    //生活购物
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3392&region=&page='=>array('page'=>3,'cate'=>'生活购物'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3476&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3382&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3385&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3426&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3393&region=&page='=>array('page'=>1,'cate'=>'生活购物'),
    

    //影音娱乐
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3432&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
    */
    /*
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3397&region=&page='=>array('page'=>1,'cate'=>'影音娱乐'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3406&region=&page='=>array('page'=>1,'cate'=>'影音娱乐'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3432&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3398&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3386&region=&page='=>array('page'=>1,'cate'=>'影音娱乐'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3395&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3398&region=&page='=>array('page'=>2,'cate'=>'影音娱乐'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3404&region=&page='=>array('page'=>1,'cate'=>'影音娱乐'),
    
    
    //社区交友
    //文化教育频道采集地址
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3372&region=&page='=>array('page'=>1,'cate'=>'文化教育'),
    

    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3378&region=&page='=>array('page'=>1,'cate'=>'文化教育'),
    

    //地方政务
    //公共名人
    'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3413&region=&page='=>array('page'=>1,'cate'=>'公共名人'),
    */
    
    //新闻阅读
    
//'http://wei.daxues.cn/index.php?m=weixin&a=index&id=1&p=' => array ('p' => 2, 'cate' => '新闻阅读' ), //综艺明星

'http://wei.daxues.cn/index.php?m=weixin&a=index&id=44&p=' => array ('p' => 15, 'cate' => '综艺明星' ), //生活购物
/*
'http://wei.daxues.cn/index.php?m=weixin&a=index&id=47&p=' => array ('p' => 99, 'cate' => '生活购物' ), //影音娱乐
'http://wei.daxues.cn/index.php?m=weixin&a=index&id=48&p=' => array ('p' => 54, 'cate' => '影音娱乐' ), //社区交友
'http://wei.daxues.cn/index.php?m=weixin&a=index&id=49&p=' => array ('p' => 21, 'cate' => '社区交友' ), //文化教育频道采集地址
'http://wei.daxues.cn/index.php?m=weixin&a=index&id=26&p=' => array ('p' => 26, 'cate' => '文化教育' )
*/
 );
    
    return $data;
}

function get($url) {
    // 初始化，返回一个handler
    $ch = curl_init ( $url );
    // 设置选项，有返回值
    curl_setopt ( $ch, CURLOPT_RETURNTRANSFER, 1 );
    // 设置选项，来源页，这意味着可以伪造referer达到某种目的
    curl_setopt ( $ch, CURLOPT_REFERER, 'http://www.baidu.com/' );
    // 设置选项，浏览器信息
    curl_setopt ( $ch, CURLOPT_USERAGENT, 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322)' );
    // 执行
    $raw = curl_exec ( $ch );
    curl_close ( $ch ); // 关闭handler
    // 输出结果
    return $raw;

}

function getDetail($url = '') {
    $con = mysql_connect ( "127.0.0.1", "root", "" );
    if (! $con) {
        die ( "连接失败" );
    }
    mysql_set_charset ( 'utf8' );
    mysql_select_db ( "caiji", $con );
    $data = getData ();
    foreach ( $data as $urlx => $pagex ) {
        $pages = range ( 1, $data [$urlx] ['p'] );
        
        foreach ( $pages as $p ) {
            $xx = $urlx . $p;
            
            $content = get ( $xx );
            $content = str_replace ( "\r\n", '', $content );
            $content = str_replace ( "\n", '', $content );
            $content = str_replace ( "\t", '', $content );
            //echo "<pre>";
            //print_r($content);
            

            //preg_match_all ( '/(http:\/\/www.vx173.com\/content-\d+-\d+-\d+.html)/', $content, $m );
            //preg_match_all ( '/img.*?src="(http:\/\/www.vx173.com\/uploadfile.*?)".*?(http:\/\/www.vx173.com\/content-\d+-\d+-\d+.html)/', $content, $m );
            

            //preg_match_all('/(index.php\?m=weixin&a=show&id=\d+)/',$content,$m);
            

            preg_match_all ( '/img.*?src="(\/Uploads.*?)".*?(index.php\?m=weixin&a=show&id=\d+)/', $content, $m );
            
            
            $pic = isset ( $m [1] ) ? $m [1] : array ();
        
            $m = isset ( $m [2] ) ? $m [2] : array ();
            

            $arrUrl = array_unique ( $m );
            
            $arrCate = $data [$urlx] ['cate'];
            
            foreach ( $arrUrl as $key => $val ) {
        
                //index.php?m=weixin&a=show&id=2315
                //http://wei.daxues.cn/index.php?m=weixin&a=show&id=2315
                $val='http://wei.daxues.cn/'.$val;
                
                $tu = 'http://wei.daxues.cn'.$pic [$key];

                /*    
                $sql = "select * from urls where url='$val'";
                
                $rs = mysql_query ( $sql );
                $pid = mysql_fetch_assoc ( $rs );
                
                $_pid = $pid ['id'];
                */
                $content = get ( $val );
                $content = str_replace ( "\r\n", '', $content );
                $content = str_replace ( "\n", '', $content );
                $content = str_replace ( "\t", '', $content );
                
                #preg_match_all('/Hao1">k.*?<img src="(.*?)">/',$content,$l);
                

                preg_match_all ( '/<img.*?src="(.*?)".*?width="225" height="225"/', $content, $l );
                
                preg_match_all ( '/公众帐号：.*?>(.*?)</', $content, $m );
                preg_match_all ( '/微信帐号：.*?>(.*?)</', $content, $n );
                preg_match_all ( '/关注度：.*?>(.*?)</', $content, $e );
                //网站地址
                preg_match_all ( '/所在地区：.*?>(.*?)</', $content, $g );
                preg_match_all ( '/收录时间：.*?>(.*?)</', $content, $h );
                
                preg_match ( '/content">(.*?)</', $content, $j );
                
        
                
                
    echo '<pre>';
    print_r ( $m[0] );
    print_r ( $n[0]);
    print_r ( $e[0]);

    print_r ( $g[0]);
    print_r ( $h[0]);
    print_r ( $j[0]);
    exit;
   
                
            
                
                $arr = array_merge ( $m [0], $n [0], $e [0], $f [0], $g [0], $h [0], $j [0], $l [1], $k [0] );
                echo "<pre>";
                print_r($arr);
                exit;
                @$arrr = str_replace ( 'style="font-size:14px; line-height: 30px;"> ', "", $arr [6] );
                $arr [6] = str_replace ( '<', "", $arrr );
                if($tu != 'http://wei.daxues.cn/Uploads/2013/06/20/51c2fd5c59e8d.png'){
                        array_push ( $arr, $tu );  
                }
            
                
                $xulie = serialize ( $arr );
                //数据过滤
                $con = mysql_connect ( "127.0.0.1", "root", "" );
                if (! $con) {
                    die ( "连接失败" );
                }
                mysql_set_charset ( 'utf8' );
                mysql_select_db ( "caiji", $con );
                
                //foreach($arr as $val){
                $sql = "insert into data (idx,data,fk_cate) values ('$_pid','$xulie','$arrCate');";
                $cn = mysql_query ( $sql );
                
            //  }
            

            }
        }
    }
    
// exit;
}

/*
 *
 *
 *
 */

getDetail ();
exit ();
$url = 'http://www.vx173.com/index.php?m=content&c=index&a=lists&catid=6&industry=3376&region=&page=1';

$content = get ( $url );
$content = str_replace ( "\r\n", '', $content );
$content = str_replace ( "\n", '', $content );
$content = str_replace ( "\t", '', $content );

preg_match_all ( '/img.*?src="(http:\/\/www.vx173.com\/uploadfile.*?)".*?(http:\/\/www.vx173.com\/content-\d+-\d+-\d+.html)/', $content, $m );

echo '<pre>';

print_r ( $m );

exit ();
//while ((flag ===0))
//select * from table where flag=0 limit 500;
//foreach
//un
//insert table
//update old table
//foreach


//while end


$xx = 'a:7:{s:10:"ToUserName";s:15:"gh_136e18869a1c";s:12:"FromUserName";s:28:"odJ_yjkd1Re7RyzaC8TZhqub5FbU";s:10:"CreateTime";s:10:"1394514427";s:7:"MsgType";s:5:"event";s:5:"Event";s:4:"SCAN";s:8:"EventKey";s:4:"8888";s:6:"Ticket";s:96:"gQGk8DoAAAAAAAAAASxodHRwOi8vd2VpeGluLnFxLmNvbS9xLzFuVXJzeVhsNkhXUlEyYzRuMW13AAIEFYkeUwMEAAAAAA==";}a:7:{s:10:"ToUserName";s:15:"gh_136e18869a1c";s:12:"FromUserName";s:28:"odJ_yjkd1Re7RyzaC8TZhqub5FbU";s:10:"CreateTime";s:10:"1394514427";s:7:"MsgType";s:5:"event";s:5:"Event";s:4:"SCAN";s:8:"EventKey";s:4:"8888";s:6:"Ticket";s:96:"gQGk8DoAAAAAAAAAASxodHRwOi8vd2VpeGluLnFxLmNvbS9xLzFuVXJzeVhsNkhXUlEyYzRuMW13AAIEFYkeUwMEAAAAAA==";}';
$xx = unserialize ( $xx );
echo '<pre>';
print_r ( $xx );