给webmagic添加handler处理下载图片等

//添加接口

package us.codecraft.webmagic.handler;

import java.util.Map;

import org.apache.http.HttpResponse;

import us.codecraft.webmagic.Request;

public interface Handler {
    Map<String, Object> getHandlerInfo(); 
    void   handResponse(Request request, HttpResponse httpResponse);
}

 

//负责图片下载的

package us.codecraft.webmagic.handler;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import org.apache.http.HttpResponse;
import us.codecraft.webmagic.Request;

public class GmapHandler implements  Handler{

    Map<String, Object> info = null ;
    @Override
    public Map<String, Object> getHandlerInfo() {
        return info;
    }
    
public GmapHandler create(String... args) {
    // TODO Auto-generated method stub
     if (null == info) {
         info = new HashMap<String, Object>();
     }
     //关于异常--
     info.put("localpath", args[0]);
     info.put("filename",  args[1]);
     return this;
}

    @Override
    public void handResponse(Request request, HttpResponse httpResponse) {
        try {
            InputStream in = httpResponse.getEntity().getContent();
            Map<String, Object> infoMap = request.getHandler().getHandlerInfo();
            savePicToDisk(in, (String)infoMap.get("localpath"), (String)infoMap.get("filename"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static void savePicToDisk(InputStream in, String dirPath,
            String fileName) {
        try {
            File dir = new File(dirPath);
            if (dir == null || !dir.exists()) {
                dir.mkdirs();
            }
            String realPath = dirPath + fileName;
            File file = new File(realPath);
            if (file == null || !file.exists()) {
                file.createNewFile();
            }

            FileOutputStream fos = new FileOutputStream(file);
            byte[] buf = new byte[1024];
            int len = 0;
            while ((len = in.read(buf)) != -1) {
                fos.write(buf, 0, len);
            }
            fos.flush();
            fos.close();

        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                in.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    
}

 

 

//Request.java 添加 handler成员

private Handler handler=null;

public Handler getHandler() {
return handler;
}

public Request(String url,Handler handler) {
this.url = url;
this.handler = handler;
}

//Spider.java中也添加这一接口

public Spider addFileDownUrl(String url,Handler handler) {
addRequest(new Request(url,handler));
signalNewUrl();
return this;
}

//HttpClientDownloader 最重要的

protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException {
Page page = new Page();
if(request.getHandler()!=null){
request.getHandler().handResponse(request, httpResponse);
page.setRawText("");
}else{
String content = getContent(charset, httpResponse);
page.setRawText(content);
}

page.setUrl(new PlainText(request.getUrl()));
page.setRequest(request);
page.setStatusCode(httpResponse.getStatusLine().getStatusCode());
return page;
}

使用下载

package us.codecraft.webmagic.gmap;

import java.util.Random;

public class GmapImage {
    private   Integer lv;
    private   Integer row;
    private   Integer col;
    
    private   String  fileName;
    private   String  localPath;
    private   String  remoteUrl;
    
    protected final static String OS_SEP = "//";
    protected final static String WEB_SEP = "/";
    
    
    public void Create(Integer lv, Integer row, Integer col) {
        // TODO Auto-generated method stub
        this.lv  = lv;
        this.row = row;
        this.col = col;    
        createDownloadInfo();
    }

    void createDownloadInfo(){
        Random rand = new Random();
        int port = rand.nextInt(4);
        Integer x = col;
        Integer y = row;
        Integer z = lv;
        //remoteUrl = "http://mt"+port+".google.cn/vt/v=w2.115&hl=en&x="+x+"&s=&y="+y+"&z="+z;
        
        localPath = OS_SEP+z+OS_SEP+y+OS_SEP;
        fileName = "Map_x=" + x + "y="+y+"zoom="+z+".png";
        remoteUrl = "http://localhost:8080/map/"+z+WEB_SEP+y+WEB_SEP+fileName;
    }
/*
 * * 测试JavaDoc 
 * 
 * @author xiejinxiong 
 *  
 * */
    
    public String getFileName() {
        // TODO Auto-generated method stub
        return fileName;
    }

    public String getRemoteUrl() {
        // TODO Auto-generated method stub
        return remoteUrl;
    }


    public String getLocalPath() {
        // TODO Auto-generated method stub
        return localPath;
    }
}

 

 

package us.codecraft.webmagic.gmap;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;

import org.apache.http.HttpHost;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.handler.GmapHandler;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;

public class GmapPageProcessor implements PageProcessor {

    private Logger logger = LoggerFactory.getLogger(getClass());
    
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
   
    @Override
    public void process(Page page) {
    }

    @Override
    public Site getSite() {
        return site;
    }
    
    public static void main(String[] args) {
        //下载失败的,to thi tha 
        System.err.println("come!");
        Spider mspider = Spider.create(new GmapPageProcessor());        
        addMappicUrls(mspider,"D://lwomap//error.log","D://lwomap//test");
        mspider.thread(5).run();
    }
    
    private static void addMappicUrls(Spider mspider, String logfile,
            String picpath) {
        try {
            StringBuffer sb   = new StringBuffer("");
            FileReader reader = new FileReader(logfile);
            BufferedReader br = new BufferedReader(reader);
            String str = null;
            while ((str = br.readLine()) != null) {
                System.out.println(str);
                String[] spara = str.split(",");
                if (spara.length == 3) {
                    Integer lv  = Integer.parseInt(spara[0]);
                    Integer row = Integer.parseInt(spara[1]);
                    Integer col = Integer.parseInt(spara[2]);
                    GmapImage img = new GmapImage();
                    img.Create(lv, row, col);
                    mspider.addFileDownUrl(img.getRemoteUrl(),
                            new GmapHandler().create(picpath + img.getLocalPath(),img.getFileName()));
                    System.out.println(img.getRemoteUrl() + ","+ picpath + img.getLocalPath() + ","+ img.getFileName());
                } else {}
            }
            br.close();
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}