代码连接
整体流程：
- 1.输入本地文件的绝对路径，读取文件，返回String变量filestring
- 2.将filestring分词，并统计词组的词频
- 3.根据词的MD5值，求出每个词的hash值，进而求出文本的simhash值
- 4.根据simhash求出海明距离并求出相似度
- 5.将结果保存在本地
项目结构：
单元测试
测试覆盖率
单元测试代码
public class cipin_and_simhashTest {
//测试分词器文本查看能否成功对文本分词;
@Test
public void getTextDef() throws IOException {
String s = "测试分词器文本//.查看能否成功对文本分词";
Map<String,Integer> wF1= cipin_and_simhash.getTextDef(s);
System.out.println(wF1);
}
//多重复词句分词测试
@Test
public void getTextDef2() throws IOException {
String s = "重复的测试文本，重复的测试文本，重复的测试文本，重复的测试文本，重复的测试文本，重复的测试文本，重复的测试文本。";
Map<String,Integer> wF1= cipin_and_simhash.getTextDef(s);
System.out.println(wF1);
}
//获取SIMHASH值
@Test
public void simHash() throws IOException {
String[] s = {"测试","计","算si","m","hash","的文本//",".查看","能否","成功求","出文本","的","simhash值;"};
for(String s1 :s) {
Map<String, Integer> wF1 = cipin_and_simhash.getTextDef(s1);
String sim = cipin_and_simhash.simHash(wF1);
System.out.println(sim);
}
}
public class HaiMingTest {
//计算海明距离
@Test
public void getHaiMing() {
String str1 = "11000111001";
String str2 = "00110011100";
int d = HaiMing.getHaiMing(str1,str2);
System.out.println("海明距离为："+d);
}
//测试不同长度的simhash的海明距离
@Test
public void getHaiMing2() {
String str1 = "11000111001";
String str2 = "0011001110";
int d = HaiMing.getHaiMing(str1,str2);
System.out.println("海明距离为："+d);
}
//测试相似度
@Test
public void getSSIM() {
String str1 = "11000111001";
String str2 = "00110011100";
double s = HaiMing.getSSIM(str1,str2);
System.out.println("相似度为："+s);
}
}
public class HashTest {
//空字符窜获取hash测试
@Test
public void getHash1() throws FileException {
String str =null;
String str2= Hash.getHash(str);
System.out.println(str2);
}
@Test
public void getHash2() {
String[] str = {"这是", "一次", "获得", "哈希", "值", "的", "测试"};
for (String s : str) {
String hash = Hash.getHash(s);
System.out.println(hash.length());
System.out.println(hash);
}
}
}
public class wenjian_IOTest {
//读文件测试
@Test
public void read() throws FileException {
String str = "C:\\Users\\MZ\\Desktop\\Testfile\\orig.txt" ;
String a = wenjian_IO.read(str);
System.out.println(a);
}
//错误路径读入测试
@Test
public void read2() throws FileException {
String str = "C:/Users/MZ/Desktop//Desktop//null.txt";
String b = wenjian_IO.read(str);
System.out.println(b);
}
//空白文件读入测试
@Test
public void read3() throws FileException {
String str = null;
String c= wenjian_IO.read(str);
System.out.println(c);
}
@Test
public void writeFile() throws IOException {
double ssim = 1;
String str = "C:/Users/MZ/Desktop//Testfile/writefiletest.txt" ;
boolean a;
a = wenjian_IO.writeFile(str,ssim);
System.out.println(a);
}
@Test
public void writeFile2() throws IOException {
double ssim = 1;
String str ="p:/null.text";
boolean a = wenjian_IO.writeFile(str,ssim);
System.out.println(a);
}
@Test
public void writeFile3() throws IOException {
double ssim = 1;
String str = null;
boolean a =wenjian_IO.writeFile(str,ssim);
System.out.println(a);
}
}
public class mainTest {
// 测试原文件与add文件相似度
@Test
public void maintestADD() throws FileException, IOException {
String a = "C:\\Users\\MZ\\Desktop\\Testfile\\orig.txt" ;
String b = "C:\\Users\\MZ\\Desktop\\Testfile\\orig_0.8.add.txt";
String str = "C:\\Users\\MZ\\Desktop\\Testfile\\testadd.txt";
Map<String,Integer> wF1 = cipin_and_simhash.getTextDef(a);
Map<String,Integer> wF2 = cipin_and_simhash.getTextDef(b);
//Similarity
double s = HaiMing.getSSIM(cipin_and_simhash.simHash(wF1), cipin_and_simhash.simHash(wF2));
System.out.println(s);
wenjian_IO.writeFile(str,s);
}
// 测试原文件与del文件相似度
@Test
public void maintestDEL() throws FileException, IOException {
String a = wenjian_IO.read("C:/Users/MZ/Desktop/Testfile/orig.txt");
String b = wenjian_IO.read("C:/Users/MZ/Desktop/Testfile/orig_0.8_del.txt");
String str = "C:/Users/MZ/Desktop/Testfile/testdel.txt";
Map<String,Integer> wF1 = cipin_and_simhash.getTextDef(a);
Map<String,Integer> wF2 = cipin_and_simhash.getTextDef(b);
//Similarity
double s = HaiMing.getSSIM(cipin_and_simhash.simHash(wF1), cipin_and_simhash.simHash(wF2));
System.out.println(s);
wenjian_IO.writeFile(str,s);
}
// 测试原文件与dis_1文件相似度
@Test
public void mainteestDIS_1() throws FileException, IOException {
String a = wenjian_IO.read("C:/Users/MZ/Desktop/Testfile/orig.txt");
String b = wenjian_IO.read("C:/Users/MZ/Desktop/Testfile/orig_0.8_dis_1.txt");
String str = "C:/Users/MZ/Desktop/Testfile/testdis_1.txt";
Map<String,Integer> wF1 = cipin_and_simhash.getTextDef(a);
Map<String,Integer> wF2 = cipin_and_simhash.getTextDef(b);
//Similarity
double s = HaiMing.getSSIM(cipin_and_simhash.simHash(wF1), cipin_and_simhash.simHash(wF2));
System.out.println(s);
wenjian_IO.writeFile(str,s);
}
// 测试原文件与dis_10文件相似度
@Test
public void mainteestDIS_10() throws FileException, IOException {
String a = wenjian_IO.read("C:/Users/MZ/Desktop/Testfile/orig.txt");
String b = wenjian_IO.read("C:/Users/MZ/Desktop/Testfile/orig_0.8_dis_10.txt");
String str = "C:/Users/MZ/Desktop/Testfile/testdis_10.txt";
Map<String,Integer> wF1 = cipin_and_simhash.getTextDef(a);
Map<String,Integer> wF2 = cipin_and_simhash.getTextDef(b);
//Similarity
double s = HaiMing.getSSIM(cipin_and_simhash.simHash(wF1), cipin_and_simhash.simHash(wF2));
System.out.println(s);
wenjian_IO.writeFile(str,s);
}
// 测试原文件与dis_15文件相似度
@Test
public void mainteestDIS_15() throws FileException, IOException {
String a = wenjian_IO.read("C:/Users/MZ/Desktop//Testfile//orig.txt");
String b = wenjian_IO.read("C:/Users/MZ/Desktop//Testfile//orig_0.8_dis_15.txt");
String str = "C:/Users/MZ/Desktop/Testfile/testdis_15.txt";
Map<String,Integer> wF1 = cipin_and_simhash.getTextDef(a);
Map<String,Integer> wF2 = cipin_and_simhash.getTextDef(b);
//Similarity
double s = HaiMing.getSSIM(cipin_and_simhash.simHash(wF1), cipin_and_simhash.simHash(wF2));
System.out.println(s);
wenjian_IO.writeFile(str,s);
}
}
算法分析
相似度计算公式
用Jaccard计算文本的相似性
其中（A ∪ B）=simhash的布尔值长+海明距离；（A ∩ B）=simhash的布尔值长-海明距离；
算法设计思路
- 处理文本，分词，计算词频；
- 计算simhash，计算中的加权就为词频；
- 求出相似度
具体实现
//Word segmentation and word frequency statistics are saved in HashMap
public static Map getTextDef(String text) throws IOException {
Map<String, Integer> wordsFren=new HashMap<String, Integer>();//使用HASGHMAP 记录分词结果
IKSegmenter ikSegmenter = new IKSegmenter(new StringReader(text), true);
Lexeme lexeme;
while ((lexeme = ikSegmenter.next()) != null) {
if(lexeme.getLexemeText().length()>1){
if(wordsFren.containsKey(lexeme.getLexemeText())){
wordsFren.put(lexeme.getLexemeText(),wordsFren.get(lexeme.getLexemeText())+1);
}else {
wordsFren.put(lexeme.getLexemeText(),1);
}
}
}
return wordsFren;
}
//就hash值
public static String getHash(String str)
{
try {//Get the hash value with the MD5 value of the file
MessageDigest MD = MessageDigest.getInstance("MD5");
String BI= new BigInteger(1, MD.digest(str.getBytes("UTF-8"))).toString(2);
return BI;
} catch (Exception e) {
e.printStackTrace();
return str;
}
}
//Calculate the simhash value of the text
public static String simHash(Map<String,Integer> wordsFrenMaps){
int[] a = new int[128];
String simhash = " ";
// 获取迭代器
Iterator<Map.Entry<String, Integer>> wordsFrenMapsIterator = wordsFrenMaps.entrySet().iterator();
while (wordsFrenMapsIterator.hasNext()) {
//Map.Entry里有相应的getKey和getValue方法，能够从一个项中取出Key和Value。
Map.Entry<String, Integer> wordsFrenEntry = wordsFrenMapsIterator.next();
String WordHash = Hash.getHash(wordsFrenEntry.getKey());
if (WordHash.length() < 128) {
// hash值可能少于128位，在低位以0补齐
int dif = 128 - WordHash.length();
for (int j = 0; j < dif; j++) {
WordHash += "0";
}
}
//加权、合并
for (int j = 0; j < a.length; j++) {
if (WordHash.charAt(j) == '1') {
a[j] += wordsFrenEntry.getValue();//hans值为1加权重值，权重等于词频
} else {
a[j] -= wordsFrenEntry.getValue();//hans值为0减权重值，权重等于词频
}
}
}
//降维
for (int j = 0; j < a.length; j++) {
if (a[j] > 0) {
simhash += "1";
} else {
simhash += "0";
}
}
return simhash;
}
//Calculate the Hamming distance of two simhashes
public static int getHaiMing(String SH1 , String SH2)
{
int distance = 0;
//字符串相同才能计算海明距离
if(SH1.length() == SH2.length()){
for(int i=0;i<SH1.length();i++){
if(SH1.charAt(i)!=SH2.charAt(i)){distance++;}
}
}
else{distance = -1;}
return distance;
}
//Output the similarity of two simhash and the corresponding simhash value;
public static double getSSIM(String SH1 , String SH2){
int distance = getHaiMing(SH1,SH2);
int i=(SH1.length()-distance);
int j=(SH1.length()+distance);
return 100*i/j;//Jaccard系数计算相似度
}
运行结果
命令行运行结果
各文件与原文件相似度对比结果
JProfile进行性能分析
uploading-image-367782.png
类的内存消耗
堆内存情况
PSP表格
| ＰＳＰ各个阶段 | 自己预估时间（分钟） | 实际的记录时间（分钟） |
|---|---|---|
| 计划: 明确需求和其他因素，估计以下的各个任务需要多少时间 | ３０ | ２０ |
| 开发 （包括下面 8 项子任务） | ６００ | ６６０ |
| 需求分析 (包括学习新技术、新工具的时间) | ５０ | １２０ |
| 生成设计文档 | ２０ | ３０ |
| 设计复审 | ２０ | ２０ |
| 代码规范 (为目前的开发制定或选择合适的规范) | ３０ | ３０ |
| 具体设计 | ４０ | ７０ |
| 具体编码 | ２２０ | ２６０ |
| 代码复审 | ２０ | １０ |
| 测试（自我测试，修改代码，提交修改） | １３０ | １２０ |
| 报告 | ６０ | ８０ |
| 测试报告 | ２０ | ２０ |
| 计算工作量 | １０ | １０ |
| 事后总结, 并提出过程改进计划 | ２０ | ３０ |
| 合计 | ７３０ | ８２０ |
总结
学习了如何打包jar包，jar的使用，单元测试的设计和GitHub上传使用。学习了一个工程设计的大概步骤。其中最重要的就是单元测试的设计，可以很好的帮我找出代码的ｂｕｇ，并测试代码的可行性，通过单元测试更好的实现作业设计要求。
posted on
浙公网安备 33010602011771号