本次实验为结对实验，我和蒲煜凡完成整体实验，实验过程分工为，我进行数据爬取，蒲煜凡进行热词云构建。实验过程较为顺利。

我所负责部分遇到问题，爬取出现数据库相关问题，经分析，是爬取的数据过长，而数据库设计的长度不足。调整数据库字段长度后，问题解决。

代码如下

Yiqing

package com.jdbc.bean;

 

public class Yiqing {

private String title;

 

public Yiqing(String title) {

super();

this.title = title;

}

public String getTitle() {

return title;

}

 

public void setTitle(String title) {

this.title = title;

}

 

}

yiqidao

package com.jdbc.dao;

 

import java.sql.Connection;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

import java.util.ArrayList;

import java.util.List;

 

import com.jdbc.bean.Yiqing;

import com.jdbc.util.BaseConnection;

 

public class yiqidao {

 

public static List<Yiqing> find2 ()

{

List<Yiqing> list = new ArrayList<Yiqing>();

Connection conn = BaseConnection.getConnection();

Statement statement = null;

String sql = "SELECT * FROM lunwen ";  

        ResultSet rs = null;

        try {

         statement = conn.createStatement();

         rs = statement.executeQuery(sql);

         Yiqing yiqing = null;

         while(rs.next())

         {

         String title = rs.getString(4);

         yiqing = new Yiqing(title);

         list.add(yiqing);

         } 

        }catch (SQLException e) {

         e.printStackTrace();

         }finally

     {

     BaseConnection.close(rs, statement, conn);

     }

     return list;

        }

}

yiqingServlet

package com.jdbc.servlet;

 

import java.io.IOException;

import java.util.List;

 

import javax.servlet.ServletException;

import javax.servlet.annotation.WebServlet;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

 

import com.jdbc.bean.Yiqing;

import com.jdbc.dao.yiqidao;

 

 

@WebServlet("/yiqingServlet")

public class yiqingServlet extends HttpServlet {

private static final long serialVersionUID = 1L;

 

    public  yiqingServlet() {

        super();

    }

protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

response.setContentType("text/html;charset=UTF-8");

request.setCharacterEncoding("UTF-8");

 

String method = request.getParameter("method");

//System.out.print(method);

if(method.equals("pc"))

{

add(request,response);

}

}

private void add(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

response.setContentType("text/html;charset=UTF-8");

request.setCharacterEncoding("UTF-8");

List<Yiqing> list =  yiqidao.find2();

System.out.println(list);

request.setAttribute("list", list);

request.getRequestDispatcher("ciyuntu.jsp").forward(request,response);

}

protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

// TODO Auto-generated method stub

doGet(request, response);

 

}

}

BaseConnection

package com.jdbc.util;

 

 

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

 

public class BaseConnection {

 

 public static Connection getConnection(){

      Connection conn=null;

      String driver = "com.mysql.cj.jdbc.Driver";

      String url = "jdbc:mysql://localhost:3306/mydate?serverTimezone=UTC&characterEncoding=utf8&useSSL=true";

      String user = "root";

      String password = "123456";

      try{

      Class.forName(driver);

      conn=DriverManager.   

      getConnection(url,user,password);

      }catch(Exception e){

      e.printStackTrace();

      }

      return conn;

     }

 

 public static void close (Statement state, Connection conn) {

if (state != null) {

try {

state.close();

} catch (SQLException e) {

e.printStackTrace();

}

}

 

if (conn != null) {

try {

conn.close();

} catch (SQLException e) {

e.printStackTrace();

}

}

}

 

public static void close (ResultSet rs, Statement state, Connection conn) {

if (rs != null) {

try {

rs.close();

} catch (SQLException e) {

e.printStackTrace();

}

}

 

if (state != null) {

try {

state.close();

} catch (SQLException e) {

e.printStackTrace();

}

}

 

if (conn != null) {

try {

conn.close();

} catch (SQLException e) {

e.printStackTrace();

}

}

}

}

Lwpq：

import requests

import pymysql

from bs4 import BeautifulSoup

 

db = pymysql.connect('127.0.0.1',

                     port=3306,

                     user='root',

                     password='mysjz',

                     db='qsly',

                     charset='utf8')

 

cursor = db.cursor()

 

headers={

        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"

        }

url="http://openaccess.thecvf.com/CVPR2019.py"

html=requests.get(url)

 

soup=BeautifulSoup(html.content,'html.parser')

 

soup.a.contents=='pdf'

 

pdfs=soup.findAll(name="a",text="pdf")

 

lis = []

jianjie=""

for i,pdf in enumerate(pdfs):

    pdf_name=pdf["href"].split('/')[-1]

    name=pdf_name.split('.')[0].replace("_CVPR_2019_paper","")

    link="http://openaccess.thecvf.com/content_CVPR_2019/html/"+name+"_CVPR_2019_paper.html"

    url1=link

    html1 = requests.get(url1)

    soup1 = BeautifulSoup(html1.content, 'html.parser')

    weizhi = soup1.find('div', attrs={'id':'abstract'})

    if weizhi:

        jianjie =weizhi.get_text();

    print("ok")

    info = {}

    info['title'] = name

    info['link'] =link

    info['abstract']=jianjie

 

    lis.append(info)

 

cursor = db.cursor()

for i in range(len(lis)):

    cols = ", ".join('`{}`'.format(k) for k in lis[i].keys())

    print(cols)  # '`name`, `age`'

 

    val_cols = ', '.join('%({})s'.format(k) for k in lis[i].keys())

    print(val_cols)  # '%(name)s, %(age)s'

 

    sql = "insert into lunwen(%s) values(%s)"

    res_sql = sql % (cols, val_cols)

    print(res_sql)

 

    cursor.execute(res_sql, lis[i])  # 将字典a传入

    db.commit()

    print("ok")

 

 

 实验结果如下：

数据库界面

 

 

web界面