hadoop环境搭建

Master 192.168.56.130
slave1 192.168.56.131
slave2 192.168.56.132

首先安装jdk

[root@Master ~]# sh jdk-6u22-linux-x64.bin

[root@Master ~]# mv jdk1.6.0_22 /usr/local/jdk

[root@Master ~]# vi /etc/profile
//添加
JAVA_HOME=/usr/local/jdk/
export JAVA_HOME
CLASSPATH=/usr/local/jdk/lib/tools.jar
export CLASSPATH
PATH=/usr/local/jdk/bin:$PATH
export PATH

安装hadoop
[root@Master ~]# tar xvf hadoop-1.0.3.tar.gz

[root@Master ~]# mv hadoop-1.0.3 /usr/local/hadoop

[root@Master ~]# vi /etc/profile
//添加
# set hadoop path
export HADOOP_HOME=/usr/local/hadoop
export PATH=/usr/local/hadoop/bin:$PATH

[root@Master ~]# source /etc/profile

[root@Master ~]# mkdir /usr/local/hadoop/tmp

[root@Master ~]# vi /usr/local/hadoop/conf/hadoop-env.sh
//添加
# set java environment
export JAVA_HOME=/usr/local/jdk

配置core-site.xml文件
[root@Master ~]# vi /usr/local/hadoop/conf/core-site.xml
//修改
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/tmp</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://192.168.56.130:9000</value>
</property>
</configuration>

配置hdfs-site.xml文件

修改Hadoop中HDFS的配置，配置的备份方式默认为3
[root@Master ~]# vi /usr/local/hadoop/conf/hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>

</property>
</configuration>

配置mapred-site.xml文件
修改Hadoop中MapReduce的配置文件，配置的是JobTracker的地址和端口
[root@Master ~]# vi /usr/local/hadoop/conf/mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
<property>
<name>mapred.job.tracker</name>
<value>http://192.168.56.130:9001</value>
</property>
</configuration>

[root@Master ~]# vi /usr/local/hadoop/conf/masters
192.168.56.130 //本机的IP

[root@Master ~]# vi /usr/local/hadoop/conf/slaves
192.168.56.131
192.168.56.132

//只需一次，下次启动不再需要格式化，只需 start-all.sh
[root@Master local]# hadoop namenode -format
Warning: $HADOOP_HOME is deprecated.

14/01/03 14:01:58 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG: host = Master.hadoop/192.168.56.130
STARTUP_MSG: args = [-format]
STARTUP_MSG: version = 1.0.3
STARTUP_MSG: build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.0 -r 1335192; compiled by
'hortonfo' on Tue May 8 20:31:25 UTC 2012
************************************************************/
14/01/03 14:01:58 INFO util.GSet: VM type = 64-bit
14/01/03 14:01:58 INFO util.GSet: 2% max memory = 19.33375 MB
14/01/03 14:01:58 INFO util.GSet: capacity = 2^21 = 2097152 entries
14/01/03 14:01:58 INFO util.GSet: recommended=2097152, actual=2097152
14/01/03 14:01:58 INFO namenode.FSNamesystem: fsOwner=root
14/01/03 14:01:58 INFO namenode.FSNamesystem: supergroup=supergroup
14/01/03 14:01:58 INFO namenode.FSNamesystem: isPermissionEnabled=true
14/01/03 14:01:58 INFO namenode.FSNamesystem: dfs.block.invalidate.limit=100
14/01/03 14:01:58 INFO namenode.FSNamesystem: isAccessTokenEnabled=false accessKeyUpdateInterval=0 min(s),
accessTokenLifetime=0 min(s)
14/01/03 14:01:58 INFO namenode.NameNode: Caching file names occuring more than 10 times
14/01/03 14:01:59 INFO common.Storage: Image file of size 110 saved in 0 seconds.
14/01/03 14:01:59 INFO common.Storage: Storage directory /usr/local/hadoop/tmp/dfs/name has been successfully formatted.
14/01/03 14:01:59 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at Master.hadoop/192.168.56.130
************************************************************/

出现了一个警告，这个警告并不影响hadoop执行，把防火墙关了就可以了

[root@Master bin]# chmod 777 start-all.sh

[root@Master bin]# ./start-all.sh
Warning: $HADOOP_HOME is deprecated.

starting namenode, logging to /usr/local/hadoop/libexec/../logs/hadoop-root-namenode-Master.hadoop.out
192.168.56.132: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-root-datanode-slave2.hadoop.out
192.168.56.131: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-root-datanode-slave1.hadoop.out
192.168.56.130: starting secondarynamenode, logging to /usr/local/hadoop/libexec/../logs/hadoop-root-secondarynamenode-
Master.hadoop.out
starting jobtracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-root-jobtracker-Master.hadoop.out
192.168.56.131: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-root-tasktracker-
slave1.hadoop.out
192.168.56.132: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-root-tasktracker-
slave2.hadoop.out

//slave1上查看
[root@slave1 tmp]# ll
总用量 8
drwxr-xr-x. 3 root root 4096 11月 25 16:31 dfs
drwxr-xr-x. 3 root root 4096 11月 25 16:31 mapred

//slave2上查看
[root@slave2 tmp]# pwd
/usr/local/hadoop/tmp
[root@slave2 tmp]# ll
总用量 8
drwxr-xr-x. 3 root root 4096 1月 6 22:31 dfs
drwxr-xr-x. 3 root root 4096 1月 6 22:31 mapred

验证hadoop
[root@Master tmp]# jps
5642 Jps
5495 JobTracker
5259 NameNode
5414 SecondaryNameNode

[root@slave1 tmp]# jps
5163 TaskTracker
5080 DataNode
5254 Jps

[root@slave2 tmp]# jps
43293 Jps
43110 DataNode
43196 TaskTracker

//第二种验证方式
[root@Master tmp]# hadoop dfsadmin -report
Warning: $HADOOP_HOME is deprecated.

Configured Capacity: 38144827392 (35.53 GB)
Present Capacity: 29948878848 (27.89 GB)
DFS Remaining: 29948809216 (27.89 GB)
DFS Used: 69632 (68 KB)
DFS Used%: 0%
Under replicated blocks: 0
Blocks with corrupt replicas: 0
Missing blocks: 0

-------------------------------------------------
Datanodes available: 2 (2 total, 0 dead)

Name: 192.168.56.131:50010
Decommission Status : Normal
Configured Capacity: 19072413696 (17.76 GB)
DFS Used: 28672 (28 KB)
Non DFS Used: 4117295104 (3.83 GB)
DFS Remaining: 14955089920(13.93 GB)
DFS Used%: 0%
DFS Remaining%: 78.41%
Last contact: Fri Jan 03 15:01:42 CST 2014

Name: 192.168.56.132:50010
Decommission Status : Normal
Configured Capacity: 19072413696 (17.76 GB)
DFS Used: 40960 (40 KB)
Non DFS Used: 4078653440 (3.8 GB)
DFS Remaining: 14993719296(13.96 GB)
DFS Used%: 0%
DFS Remaining%: 78.61%
Last contact: Fri Jan 03 15:01:44 CST 2014

目前环境只是搭建出来了，具体功能还在测试中！

posted @ 2014-01-07 10:45 kingtigerhu 阅读(204) 评论(0) 收藏举报

刷新页面返回顶部

kingtigerhu深山树林

hadoop环境搭建

公告