Hadoop集群配置以及java maven工程搭建
Hadoop配置
1.下载hadoop2.7.3
http://hadoop.apache.org/releases.html
2.安装jdk
官网下载
3. 配置java和Hadoop环境变量
vi /etc/profile 将一下代码追加到/etc/profile文件末尾
#set java environment
JAVA_HOME=/usr/local/src/java/jdk1.8.0_101
CLASSPATH=.:$JAVA_HOME/lib.tools.jar
PATH=$JAVA_HOME/bin:$PATH
export JAVA_HOME CLASSPATH PATH
#config hadoop
export HADOOP_HOME=/usr/local/src/hadoop/
export PATH=$HADOOP_HOME/bin:$PATH
#hadoop logs file path
export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
4.在Hadoop-env.sh需要手动配置jdk才能生效
vi hadoop-env.sh
追加或修改：
export JAVA_HOME=/usr/local/src/java/jdk1.8.0_101
5.配置core-site.xml 的namenode和hdfs目录
vi core-site.xml
<configuration> <property> <name>fs.defaultFS</name> <value>hdfs://node1:9000</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/usr/local/src/hadoop/tmp</value> </property> </configuration>
6.配置hdfs-site.xml 数据副本 namenode和datanode存放的数据文件夹
<configuration> <property> <name>dfs.replication</name> <value>2</value> </property> <property> <name>dfs.namenode.name.dir</name> <value>file:/usr/local/src/hadoop/tmp/dfs/name</value> </property> <property> <name>dfs.datanode.data.dir</name> <value>file:/usr/local/src/hadoop/tmp/dfs/data</value> </property> </configuration>
7.配置 mapred-site.xml
mv mapred-site.xml.template mapred-site.xml
vi mapred-site.xml
<configuration> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> </configuration>
8.配置yarn-site.xml
<configuration> <!-- Site specific YARN configuration properties --> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.resourcemanager.address</name> <value>node1:8032</value> </property> <property> <name>yarn.resourcemanager.scheduler.address</name> <value>node1:8030</value> </property> <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>node1:8031</value> </property> </configuration>
以上所有配置完 执行namenode格式化
hdfs namenode -format
9.执行hadoop自带的jar
解除namenode 安全模式
hadoop dfsadmin -safemode leave
运行hadoop自带的jar
hadoop jar /usr/local/src/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar wordcount /input /output
//hadoop运行jar流程
http://jingyan.baidu.com/album/ce09321b7a2e052bff858fd9.html?picindex=4
10.ssh免密码登录
- ssh-keygen -t rsa
- scp id_rsa.pub weekend01:home/hadoop
- touch authorized_keys
- cat 波浪线/id_rsa.pub >> authorized_keys
- cat authorized_keys
- chmod 600 authorized_keys 权限必须是600
- ssh node1 stop-yarn.sh
- cat /home/hadoop/id_rsa.pub >> authorized_keys
11.浏览器器访问hadoop服务
hadoop查看地址
http://192.168.22.131:50070/dfshealth.html#tab-datanode
http://192.168.22.131:50070/dfshealth.html#tab-datanode
12.搭建java hadoop maven工程pom.xml的配置
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.xiaot</groupId> <artifactId>bigdata</artifactId> <version>0.0.1-SNAPSHOT</version> <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.7.3</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.7.3</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>2.7.3</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-jobclient</artifactId> <version>2.7.3</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-common</artifactId> <version>2.7.3</version> </dependency> <dependency> <groupId>jdk.tools</groupId> <artifactId>jdk.tools</artifactId> <version>1.7</version> <scope>system</scope> <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath> </dependency> </dependencies> </project>
Hdfs文件上传下载java demo
package bigdata;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Progressable;
/**
* 文件管理
* @author xiaotian
*
*/
public class FileManager {
public FileSystem getFileSystem() throws Exception{
Configuration conf = new Configuration();
FileSystem dfs = FileSystem.get(new URI("hdfs//:node1:9000"), conf);
return dfs;
}
/**
* 上传文件
* @param src
* @param dst
* @throws Exception
*/
public static void uploadFile(String src,String dst) throws Exception{
InputStream in = new BufferedInputStream(new FileInputStream(src));
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(dst), conf,"root");
OutputStream os = fs.create(new Path(dst), new Progressable() {
public void progress() {
System.out.print(".");
}
});
IOUtils.copy(in, os);
System.out.println("success");
}
/**
* 删除文件
* @param dst
* @throws Exception
*/
public static void deleteFile(String dst) throws Exception{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(dst), conf,"root");
if(fs.exists(new Path(dst))){
fs.delete(new Path(dst), true);
}
}
/**
* 下载
* @param src
* @param dst
* @throws Exception
*/
public static void downloadFile(String src,String dst) throws Exception{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(src), conf,"root");
fs.copyToLocalFile(false,new Path(src), new Path(dst));
}
/**
* 遍历文件夹及所有文件
* @param dst
* @throws Exception
*/
public static void listFiles(String dst) throws Exception{
Configuration conf = new Configuration();
FileSystem dfs = FileSystem.get(URI.create(dst),conf,"root");
FileStatus[] listStatus = dfs.listStatus(new Path(dst));
for (FileStatus fileStatus : listStatus) {
if(fileStatus.isDirectory()){
System.out.println("dir:"+fileStatus.getPath().getName()+" owner:"+fileStatus.getOwner()+" Permission:"+fileStatus.getPermission());
//System.out.println(fileStatus.getPath().toString());
listFiles(fileStatus.getPath().toString());
}else{
System.out.println("file:"+fileStatus.getPath().getName()+" owner:"+fileStatus.getOwner()+" Permission:"+fileStatus.getPermission());
}
}
}
public static void main(String[] args) throws Exception {
String src = "D:\\jdk-8u101-linux-x64.tar.gz";
String dst = "hdfs://node1:9000/jdk.tar.gz";
//上传文件
//uploadFile(src,dst);
//deleteFile
//deleteFile(dst);
//dst = "D:\\test\\jdk-8u101-linux-x64.tar.gz";
//src = "hdfs://node1:9000/jdk.tar.gz";
//downloadFile(src,dst);
dst = "hdfs://node1:9000/";
listFiles(dst);
}
}
下载程序会出现问题
(null) entry in command string: null chmod 0644
解决方案：
eclipse 中运行 Hadoop2.7.3 map reduce程序 出现错误(null) entry in command string: null chmod 0700
发表: 2016-09-27 浏览: 667
环境：windows
Hadoop版本：hadoop2.7.3
开发工具：eclispe-hadoop-plugin
运行map reduce任务报错：
(null) entry in command string: null chmod 0700
解决办法：
在https://github.com/SweetInk/hadoop-common-2.7.1-bin
中下载winutils.exe,libwinutils.lib 拷贝到%HADOOP_HOME%\bin目录 。
再次执行程序，报错：
Exception in thread "main" java.lang.UnsatisfiedLinkError: org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z
at org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Native Method)
at org.apache.hadoop.io.nativeio.NativeIO$Windows.access(NativeIO.java:609)
at org.apache.hadoop.fs.FileUtil.canRead(FileUtil.java:977)
at org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:187)
at org.apache.hadoop.util.DiskChecker.checkDirAccess(DiskChecker.java:174)
at org.apache.hadoop.util.DiskChecker.checkDir(DiskChecker.java:108)
at org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext.confChanged(LocalDirAllocator.java:285)
at org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext.getLocalPathForWrite(LocalDirAllocator.java:344)
at org.apache.hadoop.fs.LocalDirAllocator.getLocalPathForWrite(LocalDirAllocator.java:150)
at org.apache.hadoop.fs.LocalDirAllocator.getLocalPathForWrite(LocalDirAllocator.java:131)
at org.apache.hadoop.fs.LocalDirAllocator.getLocalPathForWrite(LocalDirAllocator.java:115)
at org.apache.hadoop.mapred.LocalDistributedCacheManager.setup(LocalDistributedCacheManager.java:125)
at org.apache.hadoop.mapred.LocalJobRunner$Job.<init>(LocalJobRunner.java:163)
at org.apache.hadoop.mapred.LocalJobRunner.submitJob(LocalJobRunner.java:731)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:240)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308)
at com.jack.hadoop.temperature.MinTemperature.main(MinTemperature.java:37)
在https://github.com/SweetInk/hadoop-common-2.7.1-bin中下载hadoop.dll，并拷贝到c:\windows\system32目录中。
执行map reduce程序， 正常。
注意：程序中的路径都要写windows文件系统中的路径了，不再是hdfs中的路径。
Hadoop运行jar文件 以及查看运行结果
hadoop手动上传文件到指定目录
hadoop fs -put /usr/local/wc.txt /input
hadoop jar /usr/local/src/doc/wc.jar bigdata.WordCountRun /input/wc.txt /output/wc
hadoop fs -cat /output/qq1/part-r-00000
如果生命没有遗憾，没有波澜
浙公网安备 33010602011771号