Java构建Hadoop操作HDFS

Java构建Hadoop操作HDFS

1.首先在idea中创建一个Maven项目，接着在Maven项目配置pom.xml文件的依赖，这里配置父包中的依赖，使用进行管理相关依赖，并在中指定版本号

<properties>
    <maven.compiler.source>8</maven.compiler.source>
    <maven.compiler.target>8</maven.compiler.target>
    <hadoop-version>2.7.6</hadoop-version>
</properties>

<dependencyManagement>
    <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop-version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop-version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop-version}</version>
        </dependency>
    </dependencies>
</dependencyManagement>

2.接着创建子模块，进行配置pom.xml文件，引入父包的依赖，这里不需要指定版本号

<dependencies>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
    </dependency>
</dependencies>

3.这时候在子包的src目录底下创建类就可以编写代码了。

首先建立连接，Java虚拟机和Hadoop的hdfs建立连接，通过core-site.xml文件的配置建立连接

建立连接的Java文件

接着就可以在Java中使用与hdfs相关的API了。

package com.bigdate;

import javafx.scene.chart.PieChart;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Date;
//使用Java操作hdfs相关

/**
 *创建文件夹，上传文件，下载文件，查看所有文件。。。。。
 */
public class HDFSDemo {
    public static void main(String[] args) throws IOException {
        Configuration conf = new Configuration();
        //建立连接
        conf.set("fs.defaultFS","hdfs://master01:9000");
        FileSystem fileSystem = FileSystem.get(conf);
        //创建文件夹
        boolean mkdirs = fileSystem.mkdirs(new Path("/shujia/"));
        System.out.println(mkdirs);
        //上传文件
        fileSystem.copyFromLocalFile(new Path("D:\\DeskTop\\Java.txt"),new Path("/shujia/"));
        //拉取文件
        fileSystem.copyToLocalFile(new Path("/shujia/Java.txt"),new Path("D:\\DeskTop\\日常文件\\test\\Java.txt"));
        //查看所有文件信息
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/shujia/"),true);
        while (locatedFileStatusRemoteIterator.hasNext()){
            LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
            BlockLocation[] blockLocations = next.getBlockLocations();
            for (BlockLocation blockLocation : blockLocations) {
                System.out.println(blockLocation);
            }
        }
        //获取文件名称
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator1 = fileSystem.listFiles(new Path("/shujia/"),true);
        while (locatedFileStatusRemoteIterator.hasNext()) {
            LocatedFileStatus next = locatedFileStatusRemoteIterator1.next();
            String name = next.getPath().getName();
            System.out.println(name);
        }
        //获取所属用户
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator2 = fileSystem.listFiles(new Path("/shujia/"),true);
        while (locatedFileStatusRemoteIterator.hasNext()) {
            LocatedFileStatus next = locatedFileStatusRemoteIterator2.next();
            String owner = next.getOwner();
            System.out.println(owner);
        }
//        //获取权限
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator3 = fileSystem.listFiles(new Path("/shujia/"),true);
        while (locatedFileStatusRemoteIterator.hasNext()) {
            LocatedFileStatus next = locatedFileStatusRemoteIterator3.next();
            FsPermission permission = next.getPermission();
            System.out.println(permission);
        }
        //获取当前时间
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator4 = fileSystem.listFiles(new Path("/shujia/"),true);
        while (locatedFileStatusRemoteIterator1.hasNext()) {
            LocatedFileStatus next = locatedFileStatusRemoteIterator4.next();
            long modificationTime = next.getModificationTime();
            Date date = new Date(modificationTime);
            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
            String format = simpleDateFormat.format(date);
            System.out.println(format);
        }
        //以文件流进行上传与下载
        //输入流
        BufferedReader bufferedReader = new BufferedReader(new FileReader("D:\\DeskTop\\Java.txt"));
        FileInputStream fileInputStream = new FileInputStream(new File("D:\\DeskTop\\Java.txt"));
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/shujia/vb.txt"));
//        //使用Hadoop工具类进行流与流之间边读边写
        IOUtils.copy(fileInputStream,fsDataOutputStream);


        //下载文件到本地
        FSDataInputStream open = fileSystem.open(new Path("/shujia/vb.txt"));
        //输出到本地
        BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter("D:\\DeskTop\\J.txt"));
        IOUtils.copy(open,bufferedWriter);

        //释放资源
        IOUtils.closeQuietly(fileInputStream);
        IOUtils.closeQuietly(fsDataOutputStream);
    }


}