使用正则+URL实现对网页超链接提取并保存到本地

package web;
import java.io.BufferedReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@SuppressWarnings("all")
public class WebSpiderCopy {
public static void main(String[] args) {
Spider.Data("https://www.bilibili.com/","gbk","D:\\myjava\\爬虫by163.txt");
System.out.println();
}
}
@SuppressWarnings("all")
class Spider{
private String path;
public Spider(String path) {
this.path=path;
}
public static void Data(String URLdes,String type,String storage) {
String str=Spider.spider(URLdes,type);
//清洗数据，得到所有超链接
//Pattern p=Pattern.compile("(<a)([\\s\\S]{1,}?)(</a>)");
//得到超链接地址
Pattern p1=Pattern.compile("href=\"(.+?)\"");

//（）分3组，group(1)是得到你真正想要的超链接 \*是转义字符，将"转为字符"
Matcher m=p1.matcher(str);
ArrayList<String> list=new ArrayList<String>();
while(m.find())
{
list.add(m.group(1));

}
Save(list,storage);


}
public static void Save(ArrayList<String> list,String storage)
{

try {
FileWriter writer=new FileWriter(storage);

//这块可以用递归再次访问这个链接，然后再到这
//能下载所有东西
for(String s:list)
{
if(!s.equals("javascript:;"))
writer.write(s+"\n");
}

writer.flush();
writer.close();

} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}





}

//得到url源码并返回
public static String spider(String path,String type) {
StringBuilder builder =new StringBuilder();
try {
URL url=new URL(path);
//以下是伪装浏览器， 我也不咋懂 (●'◡'●)
HttpURLConnection connection=(HttpURLConnection)url.openConnection();
connection.setRequestMethod("GET");
connection.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36");
try {
//很重要啊，根据网页编码格式修改编码,防止出现乱码
InputStreamReader in=new InputStreamReader(connection.getInputStream(),type);
BufferedReader reader=new BufferedReader(in);
String str="";
while((str=reader.readLine())!=null)
{
builder.append(str);
//System.out.println(str);
}
in.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ProtocolException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
return builder.toString();
}

 

 

}