java操作HDFS

1.首先在pom文件中导入包

<dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.6</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.6</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.6</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/junit/junit -->
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.3</version>
        </dependency>

</dependencies>

2.具体实现代码

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;
public class HdfsAPI {
    FileSystem fs;
    //获取hdfs文件管理系统 file system
    @Before
    public  void main() throws Exception{
    //获取配置文件
    Configuration configuration = new Configuration();
    configuration.set("dfs.replication", "1");
    //获取（指定）连接地址
    URI uri = new URI("hdfs://master:9000");
    //创建（获取）hdfs文件管理系统的对象，通过对象操作hdfs
    fs = FileSystem.get(uri, configuration);

}
@Test
//创建目录
public void mk() throws Exception {
    boolean mkdirs = fs.mkdirs(new Path("/test/test/test"));
    System.out.println(mkdirs);
}

@Test
//删除
public void del() throws Exception {
    //boolean表示是否可以迭代删除 ture:可以 false:不可以
    boolean delete = fs.delete(new Path("/test"), true);
    System.out.println(delete);
}

@Test
public void listStatus() throws Exception {
    //查看目录下文件列表
    FileStatus[] fileStatuses = fs.listStatus(new Path("/data"));
    //获取一个一个单个文件
    for (FileStatus fs : fileStatuses) {
        System.out.println(fs.getBlockSize());//获取Block块大小 128MB会转换成字节数134217728
        System.out.println(fs.getLen()); //实际数据量
        System.out.println(fs.getReplication());//获取副本数
        System.out.println(fs.getPermission());//获取权限
        System.out.println(fs.getPath());//获取路径
        System.out.println("--------------------------------------");
    }
}

//文件切分一个block块
@Test
public void listBlockLocation() throws Exception {
    BlockLocation[] fbl = fs.getFileBlockLocations(new Path("/data/students.txt"), 0, 10);
    for (BlockLocation blockLocation : fbl) {
        String[] hosts = blockLocation.getHosts();
        for (String s : hosts) {
            System.out.println(s);
        }
        System.out.println(blockLocation.getLength());
        String[] names = blockLocation.getNames();
        for (String name : names) {
            System.out.println(name);
        }
        System.out.println(blockLocation.getOffset());
        String[] topologyPaths = blockLocation.getTopologyPaths();
        for (String s : topologyPaths) {
            System.out.println(s);
        }
    }
}

//切分多个block块
public void getBlockLocation() throws Exception {
    BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(new Path("/data/bigstudents2.txt"), 0, 1000000000);
    for (BlockLocation blockLocation : fileBlockLocations) {
        String[] hosts = blockLocation.getHosts();
        for (String s : hosts) {
            System.out.println(s);
            System.out.println(blockLocation.getOffset());
        }
    }
}

//读取文件
@Test
public void open() throws Exception {
    FSDataInputStream open = fs.open(new Path("/data/students.txt"));
    BufferedReader br = new BufferedReader(new InputStreamReader(open));
    String line;
    while ((line = br.readLine()) != null) {
        System.out.println(line);
    }
    open.close();
}
//hdfs上的文件 默认不能被修改
//想要实现修改 只能删除 重新上传（创建）
//创建一个文件
@Test
public void create() throws IOException {
    FSDataOutputStream fsDataOutputStream = fs.create(new Path("/data/students1.txt"));
    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream));
    bw.write("你好");
    bw.write("世界");
    bw.newLine();
    bw.write("helloworld");
    bw.flush();
    fsDataOutputStream.close();
}

}