java实现爬虫功能
/**
* 爬取新闻信息，封装成实体bean
*/
public class GetNews {
public List<News> getNews() {
// 存储新闻对象
List<News> list = new ArrayList<News>();
try {
// 请求DOM文档
Document document = Jsoup.connect("http://baijia.baidu.com/").get();
// 解析
String selector = "h3>a";
Elements titlels = document.select(selector);
for (Element title : titlels) {
// System.out.println("标题---" + title.text());
// 再次请求a标签，获取内容
String url = title.absUrl("href");
Document document1 = Jsoup.connect(url).get();
String selectTime = document1.select("span[class=time]").text();
// System.out.println("时间---" + selectTime);
String selectBody = document1.select(
"div[class=article-detail]").text();
// System.out.println("正文---" + selectBody);
// 构成news对象加入list集合
News news = new News();
news.setTitle(title.text());
news.setBody(selectBody);
news.setDate(selectTime);
list.add(news);
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return list;
}
}
/*
* 把获得的news对象存入数据库
*/
public int save(List<News> list) {
// sql前缀
String sql = "insert into news (title,body,date) values";
/*
* 这种方式插入数据库 速度最快
*/
for (News news : list) {
sql = sql + "('" + news.getTitle() + "','" + news.getBody() + "','"
+ news.getDate() + "'),";
}
sql = sql.substring(0, sql.length() - 1);
System.out.println(sql);
int rows = BaseDao.executeUpdate(sql);
return rows;
}
/**
* 连接数据库 通用的 工具类
*
*/
public class BaseDao {
// 创建需要得到JDBC API
protected static Connection connection = null;
protected static PreparedStatement ps = null;
protected static ResultSet rs = null;
// 01.获取数据库连接
public static boolean getConnection() {
/**
* 获取数据库连接的4要素 连接数据库的前提
*/
String driver = ConfigManager.getInstance().getValue("jdbc.driver");
String url = ConfigManager.getInstance().getValue("jdbc.url");
String userName = ConfigManager.getInstance().getValue("jdbc.userName");
String password = ConfigManager.getInstance().getValue("jdbc.password");
try {
Class.forName(driver); // 加载驱动
connection = DriverManager.getConnection(url, userName, password);
} catch (ClassNotFoundException e) {
e.printStackTrace();
return false;
} catch (SQLException e) {
e.printStackTrace();
return false;
}
return true;
}
/**
* 03.增删改 executeUpdate() 返回int 代表影响数据库中的行数 delete from user; delete from
* user where id=? and name=?;
*/
public static int executeUpdate(String sql, Object... params) {
int rowNum = 0;
if (getConnection()) { // 操作数据库 肯定现有连接
try {
ps = connection.prepareStatement(sql);
// 循环给sql语句中的？占位符 赋值
for (int i = 0; i < params.length; i++) {
ps.setObject(i + 1, params[i]);
}
// 执行sql语句
rowNum = ps.executeUpdate();
} catch (SQLException e) {
e.printStackTrace();
} finally {
closeConnection(); // 关闭连接
}
}
return rowNum;
}
/**
* 04.查询 executeQuery() 返回ResultSet select * from user; select * from user
* where id=? and name=?;
*/
public static ResultSet executeQuery(String sql, Object... params) {
if (getConnection()) { // 操作数据库 肯定现有连接
try {
ps = connection.prepareStatement(sql);
// 循环给sql语句中的？占位符 赋值
for (int i = 0; i < params.length; i++) {
ps.setObject(i + 1, params[i]);
}
// 执行sql语句
rs = ps.executeQuery();
} catch (SQLException e) {
e.printStackTrace();
}
}
return rs;
}
// 02.释放资源
public static boolean closeConnection() {
// 如果对象都没有创建 ？ 能关闭吗？ 必须进行非空判断
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
return false;
}
}
if (ps != null) {
try {
ps.close();
} catch (SQLException e) {
e.printStackTrace();
return false;
}
}
if (connection != null) {
try {
connection.close();
} catch (SQLException e) {
e.printStackTrace();
return false;
}
}
return true;
}
}
/*
* 输入关键字 查询 模糊查询
*/
public List<News> selectNews(String name) {
List<News> list = new ArrayList<News>();
String sql = "select * from news where title like ?";
Object[] params = { "%" + name + "%" };
ResultSet rs = BaseDao.executeQuery(sql, params);
try {
// 遍历结果集
while (rs.next()) {
// 创建新闻对象
News news = new News();
// 获取每一行的每一列
news.setId(rs.getInt("id"));
news.setTitle(rs.getString("title"));
news.setBody(rs.getString("body"));
news.setDate(rs.getString("date"));
list.add(news);
}
} catch (Exception e) {
// TODO: handle exception
}
return list;
}
/*
* 单例 读取配置文件的工具类
* */
public class ConfigManager {
// 01.创建自身的静态对象
private static ConfigManager manager = new ConfigManager();
private static Properties properties;
// 02.私有化构造
private ConfigManager() {
// 获取配置文件的路径
String path = "jdbc.properties";
properties = new Properties();
// 创建输入流
InputStream stream = ConfigManager.class.getClassLoader()
.getResourceAsStream(path);
try {
properties.load(stream);
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
stream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
// 03.提供供外部访问的接口
public static synchronized ConfigManager getInstance() {
return manager;
}
// 提供一个 根据key取得value的方法
public static String getValue(String key) {
return properties.getProperty(key);
}
}
/*
*properties文件
*/
jdbc.url=jdbc\:mysql\://localhost\:3306/test
jdbc.userName=hhr
jdbc.password=hhr
jdbc.driver=com.mysql.jdbc.Driver
