Hadoop HDFS Web文件浏览器 - 完整部署方案
一、项目概述
开发一个Web页面，实现以下功能：
· 显示HDFS中所有文件和文件夹
· 点击文件可直接下载
· 点击文件夹可进入查看子目录
· 支持上传本地文件到HDFS指定路径
二、环境要求
组件 版本要求
JDK 1.8 或更高
Maven 3.6+
Tomcat 8.5 或 9.0
Hadoop 2.x 或 3.x
浏览器 Chrome/Firefox/Edge
已知环境： NameNode主机名 = node1，HDFS地址 = hdfs://node1:9000
三、项目代码
1. 项目结构
hadoop-web-explorer/
├── pom.xml
├── src/main/
│ ├── java/com/hadoop/web/
│ │ ├── FileListServlet.java
│ │ ├── DownloadServlet.java
│ │ └── UploadServlet.java
│ └── webapp/
│ ├── index.jsp
│ └── WEB-INF/web.xml
2. pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.hadoop</groupId>
<artifactId>hadoop-web-explorer</artifactId>
<version>1.0</version>
<packaging>war</packaging>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<hadoop.version>3.3.4</hadoop.version>
</properties>
<dependencies>
<!-- Hadoop Client -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
<!-- Servlet API -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>4.0.1</version>
<scope>provided</scope>
</dependency>
<!-- File Upload -->
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.5</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
</dependencies>
<build>
<finalName>hadoop-explorer</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
<version>3.3.2</version>
</plugin>
</plugins>
</build>
</project>
3. web.xml
<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns="http://xmlns.jcp.org/xml/ns/javaee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee
http://xmlns.jcp.org/xml/ns/javaee/web-app_4_0.xsd"
version="4.0">
<display-name>Hadoop File Explorer</display-name>
<servlet>
<servlet-name>FileListServlet</servlet-name>
<servlet-class>com.hadoop.web.FileListServlet</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>FileListServlet</servlet-name>
<url-pattern>/list</url-pattern>
</servlet-mapping>
<servlet>
<servlet-name>DownloadServlet</servlet-name>
<servlet-class>com.hadoop.web.DownloadServlet</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>DownloadServlet</servlet-name>
<url-pattern>/download</url-pattern>
</servlet-mapping>
<servlet>
<servlet-name>UploadServlet</servlet-name>
<servlet-class>com.hadoop.web.UploadServlet</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>UploadServlet</servlet-name>
<url-pattern>/upload</url-pattern>
</servlet-mapping>
<welcome-file-list>
<welcome-file>index.jsp</welcome-file>
</welcome-file-list>
</web-app>
4. FileListServlet.java
package com.hadoop.web;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.PrintWriter;
import java.net.URI;
public class FileListServlet extends HttpServlet {
// 请修改为你的NameNode地址
private static final String HDFS_URI = "hdfs://node1:9000";
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws java.io.IOException {
String path = request.getParameter("path");
if (path == null || path.trim().isEmpty()) {
path = "/";
}
response.setContentType("application/json;charset=utf-8");
PrintWriter out = response.getWriter();
Configuration conf = new Configuration();
conf.set("dfs.replication", "1");
try (FileSystem fs = FileSystem.get(URI.create(HDFS_URI), conf)) {
Path hdfsPath = new Path(path);
if (!fs.exists(hdfsPath)) {
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
out.print("{\"error\":\"Path not found: " + path + "\"}");
return;
}
FileStatus[] statuses = fs.listStatus(hdfsPath);
StringBuilder json = new StringBuilder();
json.append("[");
for (int i = 0; i < statuses.length; i++) {
FileStatus status = statuses[i];
json.append("{");
json.append("\"name\":\"").append(escapeJson(status.getPath().getName())).append("\",");
json.append("\"path\":\"").append(escapeJson(status.getPath().toString())).append("\",");
json.append("\"isDirectory\":").append(status.isDirectory()).append(",");
json.append("\"size\":").append(status.getLen()).append(",");
json.append("\"modificationTime\":").append(status.getModificationTime());
json.append("}");
if (i < statuses.length - 1) json.append(",");
}
json.append("]");
out.print(json.toString());
} catch (Exception e) {
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
out.print("{\"error\":\"" + escapeJson(e.getMessage()) + "\"}");
e.printStackTrace();
}
}
private String escapeJson(String str) {
if (str == null) return "";
return str.replace("\\", "\\\\").replace("\"", "\\\"");
}
}
5. DownloadServlet.java
package com.hadoop.web;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.BufferedInputStream;
import java.net.URI;
public class DownloadServlet extends HttpServlet {
private static final String HDFS_URI = "hdfs://node1:9000";
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws java.io.IOException {
String filePath = request.getParameter("file");
if (filePath == null || filePath.trim().isEmpty()) {
response.sendError(HttpServletResponse.SC_BAD_REQUEST, "File path required");
return;
}
Configuration conf = new Configuration();
conf.set("dfs.replication", "1");
try (FileSystem fs = FileSystem.get(URI.create(HDFS_URI), conf)) {
Path hdfsPath = new Path(filePath);
if (!fs.exists(hdfsPath) || fs.getFileStatus(hdfsPath).isDirectory()) {
response.sendError(HttpServletResponse.SC_NOT_FOUND, "File not found");
return;
}
String fileName = hdfsPath.getName();
response.setContentType("application/octet-stream");
response.setHeader("Content-Disposition", "attachment; filename=\"" +
new String(fileName.getBytes("utf-8"), "iso-8859-1") + "\"");
try (BufferedInputStream bis = new BufferedInputStream(fs.open(hdfsPath));
java.io.OutputStream os = response.getOutputStream()) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = bis.read(buffer)) != -1) {
os.write(buffer, 0, bytesRead);
}
}
} catch (Exception e) {
response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());
e.printStackTrace();
}
}
}
6. UploadServlet.java
package com.hadoop.web;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.PrintWriter;
import java.net.URI;
import java.util.List;
public class UploadServlet extends HttpServlet {
private static final String HDFS_URI = "hdfs://node1:9000";
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response)
throws java.io.IOException {
response.setContentType("text/html;charset=utf-8");
PrintWriter out = response.getWriter();
if (!ServletFileUpload.isMultipartContent(request)) {
out.print("<h3>错误: 表单必须包含 enctype='multipart/form-data'</h3>");
return;
}
DiskFileItemFactory factory = new DiskFileItemFactory();
ServletFileUpload upload = new ServletFileUpload(factory);
upload.setFileSizeMax(1024 * 1024 * 100); // 100MB限制
try {
List<FileItem> items = upload.parseRequest(request);
String targetPath = "/";
FileItem uploadedFile = null;
for (FileItem item : items) {
if (item.isFormField()) {
if ("targetPath".equals(item.getFieldName())) {
targetPath = item.getString("utf-8");
if (!targetPath.startsWith("/")) targetPath = "/" + targetPath;
if (!targetPath.endsWith("/")) targetPath = targetPath + "/";
}
} else {
uploadedFile = item;
}
}
if (uploadedFile == null || uploadedFile.getName().isEmpty()) {
out.print("<h3>错误: 请选择要上传的文件</h3>");
return;
}
String fileName = new java.io.File(uploadedFile.getName()).getName();
String hdfsFilePath = targetPath + fileName;
Configuration conf = new Configuration();
conf.set("dfs.replication", "1");
try (FileSystem fs = FileSystem.get(URI.create(HDFS_URI), conf)) {
Path hdfsPath = new Path(hdfsFilePath);
// 确保目标目录存在
Path parent = hdfsPath.getParent();
if (!fs.exists(parent)) {
fs.mkdirs(parent);
}
try (java.io.OutputStream os = fs.create(hdfsPath)) {
uploadedFile.getInputStream().transferTo(os);
}
out.print("<h3 style='color:green'>✓ 上传成功！</h3>");
out.print("<p>文件: " + fileName + "</p>");
out.print("<p>目标路径: " + hdfsFilePath + "</p>");
out.print("<a href='index.jsp?path=" + java.net.URLEncoder.encode(targetPath, "utf-8") + "'>返回文件列表</a>");
}
} catch (Exception e) {
out.print("<h3 style='color:red'>上传失败: " + e.getMessage() + "</h3>");
e.printStackTrace();
}
}
}
7. index.jsp
<%@ page contentType="text/html;charset=utf-8" language="java" %>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Hadoop HDFS 文件浏览器</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
padding: 20px;
}
.container {
max-width: 1200px;
margin: 0 auto;
background: white;
border-radius: 15px;
box-shadow: 0 20px 60px rgba(0,0,0,0.3);
overflow: hidden;
}
.header {
background: #2c3e50;
color: white;
padding: 20px 30px;
}
.header h1 { font-size: 24px; }
.upload-area {
background: #f8f9fa;
padding: 20px 30px;
border-bottom: 1px solid #ddd;
}
.upload-form {
display: flex;
gap: 15px;
align-items: flex-end;
flex-wrap: wrap;
}
.form-group {
flex: 1;
min-width: 200px;
}
.form-group label {
display: block;
margin-bottom: 5px;
color: #555;
font-weight: bold;
}
.form-group input {
width: 100%;
padding: 10px;
border: 1px solid #ddd;
border-radius: 5px;
}
.btn {
background: #27ae60;
color: white;
border: none;
padding: 10px 25px;
border-radius: 5px;
cursor: pointer;
font-weight: bold;
}
.btn:hover { background: #229954; }
.nav {
background: #ecf0f1;
padding: 15px 30px;
border-bottom: 1px solid #ddd;
}
.current-path span {
background: #3498db;
color: white;
padding: 5px 10px;
border-radius: 5px;
font-family: monospace;
}
.file-list { padding: 20px 30px; }
table {
width: 100%;
border-collapse: collapse;
}
th, td {
text-align: left;
padding: 12px;
border-bottom: 1px solid #ecf0f1;
}
th { background: #f8f9fa; }
.folder-link {
color: #e67e22;
text-decoration: none;
cursor: pointer;
font-weight: bold;
}
.file-link {
color: #3498db;
text-decoration: none;
cursor: pointer;
}
.loading { text-align: center; padding: 40px; color: #999; }
.error {
background: #e74c3c;
color: white;
padding: 15px;
border-radius: 5px;
}
.breadcrumb { margin-top: 10px; }
.breadcrumb a { color: #3498db; text-decoration: none; }
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>📁 Hadoop HDFS 文件浏览器</h1>
<p>浏览、下载、上传文件到分布式文件系统</p>
</div>
<div class="upload-area">
<form id="uploadForm" class="upload-form" enctype="multipart/form-data">
<div class="form-group">
<label>📂 目标路径 (如 /user/test/)</label>
<input type="text" id="targetPath" name="targetPath" placeholder="留空表示根目录">
</div>
<div class="form-group">
<label>📎 选择文件</label>
<input type="file" id="uploadFile" name="uploadFile">
</div>
<div class="form-group">
<button type="submit" class="btn">⬆ 上传到 HDFS</button>
</div>
</form>
<div id="uploadStatus"></div>
</div>
<div class="nav">
<div class="current-path">📍 当前位置: <span id="currentPath">/</span></div>
<div class="breadcrumb" id="breadcrumb"></div>
</div>
<div class="file-list">
<div id="fileListContent" class="loading">加载中...</div>
</div>
</div>
<script>
let currentPath = '/';
function formatSize(bytes) {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
function loadPath(path) {
currentPath = path;
document.getElementById('currentPath').innerText = path;
updateBreadcrumb(path);
document.getElementById('fileListContent').innerHTML = '<div class="loading">加载中...</div>';
fetch('/hadoop-explorer/list?path=' + encodeURIComponent(path))
.then(response => response.json())
.then(data => renderFileList(data))
.catch(error => {
document.getElementById('fileListContent').innerHTML =
'<div class="error">❌ 错误: ' + error.message + '</div>';
});
}
function updateBreadcrumb(path) {
const parts = path.split('/').filter(p => p !== '');
let html = '<a href="javascript:loadPath(\'/\')">根目录</a>';
let current = '';
for (let i = 0; i < parts.length; i++) {
current += '/' + parts[i];
html += ' / <a href="javascript:loadPath(\'' + current + '\')">' + parts[i] + '</a>';
}
document.getElementById('breadcrumb').innerHTML = html;
}
function renderFileList(files) {
if (!files || files.length === 0) {
document.getElementById('fileListContent').innerHTML =
'<div style="text-align:center;padding:40px;">📂 此文件夹为空</div>';
return;
}
let html = '<table><thead><tr><th>名称</th><th>大小</th></tr></thead><tbody>';
files.forEach(file => {
html += '<tr>';
html += '<td>';
if (file.isDirectory) {
html += '📁 <a href="javascript:loadPath(\'' + file.path + '\')" class="folder-link">' + file.name + '</a>';
} else {
html += '📄 <a href="javascript:downloadFile(\'' + file.path + '\')" class="file-link">' + file.name + '</a>';
}
html += '</td>';
html += '<td>' + (file.isDirectory ? '-' : formatSize(file.size)) + '</td>';
html += '</tr>';
});
html += '</tbody></table>';
document.getElementById('fileListContent').innerHTML = html;
}
function downloadFile(filePath) {
window.location.href = '/hadoop-explorer/download?file=' + encodeURIComponent(filePath);
}
document.getElementById('uploadForm').addEventListener('submit', function(e) {
e.preventDefault();
const targetPath = document.getElementById('targetPath').value || '/';
const file = document.getElementById('uploadFile').files[0];
if (!file) {
document.getElementById('uploadStatus').innerHTML = '<div style="color:red;">⚠ 请选择文件</div>';
return;
}
const formData = new FormData();
formData.append('targetPath', targetPath);
formData.append('uploadFile', file);
document.getElementById('uploadStatus').innerHTML = '<div style="color:#3498db;">⏳ 上传中...</div>';
fetch('/hadoop-explorer/upload', { method: 'POST', body: formData })
.then(response => response.text())
.then(html => {
document.getElementById('uploadStatus').innerHTML = html;
setTimeout(() => {
loadPath(currentPath);
document.getElementById('uploadFile').value = '';
}, 1500);
})
.catch(error => {
document.getElementById('uploadStatus').innerHTML = '<div style="color:red;">❌ 上传失败</div>';
});
});
loadPath('/');
</script>
</body>
</html>
四、部署步骤
第1步：虚拟机上安装Tomcat
# 下载Tomcat
cd /opt
sudo wget https://archive.apache.org/dist/tomcat/tomcat-9/v9.0.80/bin/apache-tomcat-9.0.80.tar.gz
# 解压
sudo tar -xzf apache-tomcat-9.0.80.tar.gz
sudo mv apache-tomcat-9.0.80 tomcat
# 启动Tomcat测试
/opt/tomcat/bin/startup.sh
# 检查是否启动成功
ps aux | grep tomcat
第2步：把War包传到虚拟机
在Windows IDEA打包后，在项目 target/ 目录找到 hadoop-explorer.war，传到虚拟机：
# 在Windows的PowerShell或CMD中执行（替换成你的虚拟机IP）
scp D:/path/to/hadoop-explorer.war root@你的虚拟机IP:/opt/tomcat/webapps/
第3步：复制Hadoop JAR包到Tomcat（重要！）
# 在虚拟机中执行
cp /opt/hadoop/share/hadoop/common/*.jar /opt/tomcat/lib/
cp /opt/hadoop/share/hadoop/common/lib/*.jar /opt/tomcat/lib/
cp /opt/hadoop/share/hadoop/hdfs/*.jar /opt/tomcat/lib/
cp /opt/hadoop/share/hadoop/hdfs/lib/*.jar /opt/tomcat/lib/
第4步：重启Tomcat
/opt/tomcat/bin/shutdown.sh
/opt/tomcat/bin/startup.sh
# 查看日志确认部署成功
tail -f /opt/tomcat/logs/catalina.out
第5步：验证部署
# 检查war是否被解压
ls /opt/tomcat/webapps/hadoop-explorer/
# 应该能看到 index.jsp, WEB-INF 等文件
第6步：Windows浏览器访问
http://node1:8080/hadoop-explorer/
可能遇到的问题
问题 解决方法
连接不上虚拟机 检查Windows hosts文件是否配置了 虚拟机IP node1
8080端口不通 虚拟机防火墙放行：sudo firewall-cmd --add-port=8080/tcp --permanent
访问HDFS失败 确认Hadoop服务运行：jps 看NameNode进程
最后验证功能
· ✅ 看到HDFS根目录文件列表
· ✅ 点击文件夹能进入
· ✅ 点击文件能下载
· ✅ 上传文件到指定路径
五、Windows客户端配置（如适用）
编辑 C:\Windows\System32\drivers\etc\hosts，添加：
虚拟机IP node1
六、功能验证清单
功能 操作 预期结果
浏览根目录 访问首页 显示HDFS根目录内容
进入文件夹 点击文件夹 显示子目录，面包屑更新
下载文件 点击文件 浏览器自动下载
上传文件 选文件+填路径 提示成功，目录刷新
七、常见问题
问题 解决方法
404 检查Tomcat是否启动，war是否正确部署
连接HDFS失败 检查Hadoop服务是否运行，执行jps查看
权限错误 chmod -R 755 /opt/tomcat/
类找不到 确认Hadoop jar包已复制到tomcat/lib
以上就是完整的部署方案，将所有代码保存到对应文件即可使用。
posted on
浙公网安备 33010602011771号