三、Hadoop 的 API

1.环境搭建

org.apache.hadoop
hadoop-hdfs
2.6.0



org.apache.hadoop
hadoop-common
2.6.0

2.问题解决

Windows开发Hadoop应用环境配置

• 解压hadoop安装包到C:/
• 将winutils.exe和hadoop.dll拷贝到hadoop的bin目录下
• 在windows配置HADOOP_HOME环境变量
• 重启开发工具idea,否则开发工具无法识别HADOOP_HOME
• 在Windows主机配置CentOS的主机名和IP的映射关系

C:\Windows\System32\drivers\etc\hosts

192.168.169.139 CentOS

HDFS权限不足导致写失败?

org.apache.hadoop.security.AccessControlException: Permission denied: user=HIAPAD, access=WRITE, inode="/":root:supergroup:drwxr-xr-x
at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkFsPermission(FSPermissionChecker.java:271)
at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:257)
...

解决方案

方案1

etc/hadoop/hdfs-site.xml

dfs.permissions.enabled
false

关闭HDFS文件权限检查,修改完成后,重启HDFS服务

方案2

-DHADOOP_USER_NAME=root

设置JAVA虚拟机启动参数java XXX -Dxx=xxx

3.HDFS

package com.baizhi.hdfs;/**

- @Author:luoht
- @Description：
- @Date：Create in 16:21 2019/1/3
*/

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.;
import java.io.*;

/**

- @program: hadoop_01
- @description:
- @author: luoht
- @create: 2019-01-03 16:21
**/

public class TestHDFS {
private FileSystem fileSystem;
private Configuration conf;

@Before
public void before() throws IOException {
conf = new Configuration();
conf.addResource("core-site.xml");
conf.addResource("hdfs-site.xml");
fileSystem=FileSystem.newInstance(conf);
}

@Test
public void testConfig(){
String value = conf.get("dfs.replication");
System.out.println(value);
}

@Test
public void testUpload01() throws IOException {
String file ="C:\Users\Administrator\Desktop\郁金香開了麽？.txt";
Path dst = new Path("/demo/access/springBoot.pdf");
FileInputStream inputStream = new FileInputStream(file);
FSDataOutputStream outputStream = fileSystem.create(dst, new Progressable() {
@Override
public void progress() {
System.out.println("..");
}
});
IOUtils.copyBytes(inputStream,outputStream,1024,true);
}

@Test
public void testUpload02() throws IOException {
Path src = new Path("C:\Users\Administrator\Desktop\郁金香開了麽？.txt");
Path dst = new Path("/郁金香开了么？.txt");
fileSystem.copyFromLocalFile(src,dst);
}

@Test
public void testDownload01() throws IOException {
String file="C:\Users\Administrator\Desktop\郁金香不开了.txt";
Path dst = new Path("/郁金香开了么？.txt");
FileOutputStream outputStream = new FileOutputStream(file);
InputStream inputStream = fileSystem.open(dst);
IOUtils.copyBytes(inputStream,outputStream,1024,true);
}

@Test
public void testDownload02() throws IOException {
Path dst = new Path("C:\Users\Administrator\Desktop\郁金香不开了.txt");
Path src = new Path("/demo/access/springBoot.pdf");
fileSystem.copyToLocalFile(false,src,dst,true);
}

@Test
public void testDelete() throws IOException {
Path src = new Path("/demo");
/true 代表递归删除子文件/
fileSystem.delete(src,true);
}

@Test
public void testExits() throws IOException {
Path src = new Path("/郁金香开了么？.txt");
boolean exists = fileSystem.exists(src);
//Junit之 断言 -静态导包
assertTrue(exists);
}

@Test
public void testMkdir() throws IOException {
Path src = new Path("/demo.assess");
boolean exists = fileSystem.exists(src);
if (!exists){
fileSystem.mkdirs(src);
}
}

@Test
public void testListFiles() throws IOException {
Path src = new Path("/tt");
RemoteIterator files = fileSystem.listFiles(src, true);
while (files.hasNext()){
LocatedFileStatus fileStatus = files.next();
System.out.println(fileStatus.getPath()+"路径 "+fileStatus.isFile()+"是否文件 "+fileStatus.getLen()+"文件大小");
System.out.println("------------------------------");
BlockLocation[] locations = fileStatus.getBlockLocations();
for (BlockLocation location : locations) {
System.out.println("offset"+location.getOffset()+"length"+location.getLength());
}
System.out.println("================================");
}
}

@Test
public void testDeleteWithTrash() throws IOException {
Trash trash = new Trash(fileSystem, conf);
Path dst = new Path("/123.txt");
trash.moveToTrash(dst);
}

@After
public void after() throws IOException{
fileSystem.close();
}

}