爬取拉勾网职位等信息（Java）

工具：火狐浏览器，selenium IDE（3版本往上），Eclipse，selenium-java.jar（需导入），selenium-server-standalone-3.141.5.jar(需导入)，poi-bin-5.2.2

package one;

import java.io.File;
import java.io.FileOutputStream;
import java.util.List;

import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Workbook;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebDriver.Navigation;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxDriver;

public class search_lagouwang {

    public static void main(String[] args) throws InterruptedException {
        
        String s = "",job="",company="",money="",experience="",whole="";
        int i=1;
        int j=1;   //i表示表格第几行，j表示网页第几页
        
        //创建工作表，需要导出到文件
        HSSFWorkbook workbook = new HSSFWorkbook();
        HSSFSheet sheet = workbook.createSheet();     
        HSSFRow row = sheet.createRow(0);
        row.createCell(0).setCellValue("职位");
        row.createCell(1).setCellValue("公司");
        row.createCell(2).setCellValue("薪资");
        row.createCell(3).setCellValue("工作经验");
        
        System.setProperty ( "webdriver.firefox.bin" , "E:\\Mozilla Firefox\\firefox.exe" );  //需导入一堆.jar
        System.setProperty("webdriver.gecko.driver", "E:\\Mozilla Firefox\\geckodriver.exe");
        //selenium3中没有火狐启动驱动，需要重新下载geckodriver.exe
        WebDriver driver = new FirefoxDriver();
        Navigation navigation = driver.navigate();
        navigation.to("https://www.lagou.com/");
        Thread.sleep(3000);
        
        driver.findElement(By.id("cboxClose")).click();  //点击弹出上面的X按钮
        Thread.sleep(3000);   //需要等待几秒，不然时间太快输入框获取不到
        driver.findElement(By.id("search_input")).clear();
        driver.findElement(By.id("search_input")).sendKeys("java");
        driver.findElement(By.id("search_button")).click();
        Thread.sleep(2000);
        //获取下一页按钮
        WebElement next = driver.findElement(By.className("lg-pagination-next"));   
        while(next != null && next.isEnabled()==true) {  //按钮存在且可点击
            //搜索过程
            
            List<WebElement> all = driver.findElements(By.className("item-top__1Z3Zo")); //是findElements，多个元素集合
            
            for(WebElement a : all) {
                //获得职位
                job = a.findElement(By.className("p-top__1F7CL"))
                        .findElement(By.tagName("a")).getText();
                //获得公司
                company = a.findElement(By.className("company-name__2-SjF"))
                        .findElement(By.tagName("a")).getText();
                //获得薪资、经验
                money = a.findElement(By.className("p-bom__JlNur"))
                        .findElement(By.tagName("span")).getText();
                whole = a.findElement(By.className("p-bom__JlNur")).getText();
                experience = whole.substring(money.length(), whole.length());
                //输出到工作表
                row = sheet.createRow(i);
                row.createCell(0).setCellValue(job);
                row.createCell(1).setCellValue(company);
                row.createCell(2).setCellValue(money);
                row.createCell(3).setCellValue(experience);
                //System.out.println("公司: "+company+"职位："+job+"薪资: "+money+"经验: "+experience);
                i++;
                
            }
            //点击下一页
            next.click();
            Thread.sleep(3000);
            j++;   //由于搜索过多页会弹出登录框，只搜素10页，j控制页数
            if(j<=10) {
                next = driver.findElement(By.className("lg-pagination-next"));
            }else {
                break;
            }            
        }
        
        
        try {
            //创建输出流，将工作表写入到文件lagouwang.xls
            File file = new File("E:/lagouwang_1.xls");
            file.createNewFile();
            FileOutputStream ot = new FileOutputStream(file);
            workbook.write(ot);
        }catch(Exception e) {
            e.printStackTrace();
        }
            
    }
    
}