<?php
namespace Home\Controller;
use Think\Controller;
class CollectController extends Controller {
//采集新闻列表包括标题,链接,简介并保存
public function index() {
$contents = file_get_contents('http://news.sohu.com/guoneixinwen.shtml'); //搜狐国内新闻首页
$contents = mb_convert_encoding($contents, "UTF-8", "gb2312");
$preg = '/maxPage = (\d+);/is';
preg_match_all($preg, $contents, $arr);
$num = $arr[1][0]; //新闻列表url最新顺序码
$preg1 = '/(上一页|下一页|尾页|末页)/';//根据页面情况写正则
preg_match_all($preg1, $contents, $arr1);
if (isset($_GET['id'])) {
if ($arr1[1] !== null && $_GET['id'] <= C('COLLECT_PAGE')) { //配置中数组采集页数
$url = 'http://news.sohu.com/guoneixinwen_' . ($num - $_GET['id']) . '.shtml'; //搜狐国内新闻分页
$id = ++$_GET['id'];
}
else {
echo '列表采集结束,将继续采集正文内容';
$this->index1();
}
}
else {
//实时新闻,避免重复,数据,图片要清空,也可以保存上次的页码,留下次继续,但比较复杂,这里是清空
$this->deleteTable('think_news'); //清空表
delFile(ROOT . '/z2/Public/Uploads/sohu/'); //清空图片
$url = 'http://news.sohu.com/guoneixinwen.shtml'; //搜狐国内新闻首页
$id = 1;
}
echo '第' . $id . '页列表采集中......';
$contents = mb_convert_encoding(file_get_contents($url), "UTF-8", "gb2312"); //转码
$preg = '/<h3><span class="com-num"><a target="_blank" href="#">comment num<\/a><\/span><a target="_blank" href="(http:\/\/.*)(?#链接)">(.*)(?#标题)<\/a><\/h3>\s*<p>(.*)(?#简介)<a target="_blank" href="http:\/\/.*">.*<\/a><\/p>/Uims';
preg_match_all($preg, $contents, $arr);
$news = M('News');
foreach ($arr[1] as $key => $value) {
$data['title'] = $arr[2][$key];
$data['url'] = $value;
$data['info'] = $arr[3][$key];
$news->create($data);
$news->add();
}
echo '<script>location.href="' . U('collect/index', array('id' => $id)) . '"</script>';
}
//根据采采集的url采集正文内容及图片并保存
public function index1() {
$news = M('News');
if (isset($_GET['cid'])) $cid = $_GET['cid'];
else $cid = 0;
$map['id'] = array('gt', $cid);
$result = $news->field('id,url')->where($map)->find();
if (null != $result) {
$contents = mb_convert_encoding(file_get_contents($result['url']), "UTF-8", "gb2312");
$preg = '/(<div itemprop="articleBody">.*)(?#正文)<!-- seo标签描述 -->/is';
preg_match_all($preg, $contents, $arr);
$contents1 = $arr[1][0]; //正文内容
if (!!$path = $this->getPath($contents1)) { //正文内的图片远程路径数组
$savePath = $this->saveImage($path); //保存图片并获取本地保存绝对路径
$contents1 = str_ireplace($path, $savePath, $contents1); //远程图片路径替换为本地绝对路径
}
$news->where('id=' . $result['id'])->setField('contents', htmlspecialchars($contents1));
echo 'id为' . $result['id'] . '的正文内容采集中......';
$cid = ++$cid;
echo '<script>location.href="' . U('collect/index1', array('cid' => $cid)) . '"</script>';
}
else {
echo '正文内容采集结束,以下是采集内容显示';
echo '<script>location.href="' . U('collect/index2') . '"</script>';
}
}
//内容显示简页
public function index2() {
$news = M('News');
$result = $news->where($map)->select();
$this->assign('aa', $result);
//$a = $this->buildHtml('1', HTML_PATH . '/collect/', APP_PATH . 'Admin/View/Login/index.html');
//echo $a;
$this->display();
}
//清空表
private function deleteTable($table) {
$sql = "TRUNCATE TABLE $table";
M()->execute($sql);
}
//获取正文内容中的远程图片路径并返回,参数采集的正文内容
private function getPath($contents) {
$path = array();
if ($contents == null) return false;
$preg = '/<img src="(http:\/\/.*)" alt=.*\/>/Uis';
if (preg_match_all($preg, $contents, $arr)) {
foreach ($arr[1] as $key => $value) {
$path[] = $value; //获取远程图片路径
}
return $path; //返回远程图片路径
}
return false;
}
//保存图片并返回本地绝对路径,参数远程图片路径数组
private function saveImage($path) {
if ($path == '') return false;
$pathArr = array();
foreach ($path as $key => $value) {
$url = $value; //远程图片路径
$filename = substr($value, strripos($value, '/')); //图片名.后缀
$savePath = './Public/Uploads/sohu' . $filename; //保存路径
ob_start(); //开启缓冲
readfile($url); //读取图片
$img = ob_get_contents(); //保存到缓冲区
ob_end_clean(); //关闭缓冲
$fp2 = @fopen($savePath, "a"); //打开本地保存图片文件
fwrite($fp2, $img); //写入图片
fclose($fp2);
$pathArr[] = 'http://localhost:9096/z2/' . str_ireplace('./', '', $savePath); //保存图片绝对路径
}
return $pathArr; //返回本地保存绝对路径
}
}
/*
z2是我的app名
相关代码
//删除文件夹里的文件,放公共函数文件里
function delFile($dir) {
$dh = opendir($dir);
while ($file = readdir($dh)) {
if ($file != "." && $file != "..") {
$fullpath = $dir . "/" . $file;
if (!is_dir($fullpath)) {
unlink($fullpath);
}
else {
deldir($fullpath);
}
}
}
closedir($dh);
}
//入口文件
define('ROOT',$_SERVER['DOCUMENT_ROOT']);
//配置文件
'COLLECT_PAGE'=> '10',//采集新闻列表页数
//图片本地保存目录'./Public/Uploads/sohu'
//index2模板代码
<volist name='aa' id='vo'>
<div style="width:800px;margin:0 auto;">
<h2>{$vo.id}.{$vo.title}</h2>
{$vo.contents|htmlString_decode}
<p>原文链接:<a href="{$vo.url}" target="blank">{$vo.url}</a></p>
</div><br><br><br><br><br><br><br>
</volist>
//表结构
--
-- `think_news`
--
CREATE TABLE IF NOT EXISTS `think_news` (
`id` int(5) unsigned NOT NULL AUTO_INCREMENT,
`title` varchar(255) DEFAULT NULL,
`url` varchar(255) DEFAULT NULL,
`info` text,
`contents` text,
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC ;
*/
