CentOS 安装 hadoop(伪分布模式)

在本机上装的CentOS 5.5 虚拟机，

软件准备：jdk 1.6 U26

hadoop:hadoop-0.20.203.tar.gz

ssh检查配置

[root@localhost~]#ssh-keygen-trsa
Generatingpublic/privatersakeypair.
Enterfileinwhichtosavethekey(/root/.ssh/id_rsa):
Createddirectory'/root/.ssh'.
Enterpassphrase(emptyfornopassphrase):
Entersamepassphraseagain:
Youridentificationhasbeensavedin/root/.ssh/id_rsa.
Yourpublickeyhasbeensavedin/root/.ssh/id_rsa.pub.
Thekeyfingerprintis:
a8:7a:3e:f6:92:85:b8:c7:be:d9:0e:45:9c:d1:36:[e-mail]
[root@localhost~]#
[root@localhost~]#cd..
[root@localhost/]#cdroot
[root@localhost~]#ls
anaconda-ks.cfgDesktopinstall.loginstall.log.syslog
[root@localhost~]#cd.ssh
[[e-mail]]#catid_rsa.pub>authorized_keys
[[e-mail]]#

[[e-mail]]#sshlocalhost
Theauthenticityofhost'localhost(127.0.0.1)'can'tbeestablished.
RSAkeyfingerprintis41:c8:d4:e4:60:71:6f:6a:33:6a:25:27:62:9b:e3:90.
Areyousureyouwanttocontinueconnecting(yes/no)?yes
Warning:Permanentlyadded'localhost'(RSA)tothelistofknownhosts.
Lastlogin:TueJun2122:40:312011
[root@localhost~]#

安装jdk

[root@localhostjava]#chmod+xjdk-6u26-linux-i586.bin
[root@localhostjava]#./jdk-6u26-linux-i586.bin
......
......
......
FormoreinformationonwhatdataRegistrationcollectsand
howitismanagedandused,see:
http://java.sun.com/javase/registration/JDKRegistrationPrivacy.html

PressEntertocontinue.....


Done.

安装完成后生成文件夹：jdk1.6.0_26

配置环境变量

[root@localhostjava]#vi/etc/profile
#添加如下信息
#setjavaenvironment
exportJAVA_HOME=/usr/java/jdk1.6.0_26
exportCLASSPATH=$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
exportPATH=$JAVA_HOME/lib:$JAVA_HOME/jre/bin:$PATH:$HOME/bin
exportHADOOP_HOME=/usr/local/hadoop/hadoop-0.20.203
exportPATH=$PATH:$HADOOP_HOME/bin

[root@localhostjava]#chmod+x/etc/profile
[root@localhostjava]#source/etc/profile
[root@localhostjava]#
[root@localhostjava]#java-version
javaversion"1.6.0_26"
Java(TM)SERuntimeEnvironment(build1.6.0_26-b03)
JavaHotSpot(TM)ClientVM(build20.1-b02,mixedmode,sharing)
[root@localhostjava]#

修改hosts

[root@localhostconf]#vi/etc/hosts
#Donotremovethefollowingline,orvariousprograms
#thatrequirenetworkfunctionalitywillfail.
127.0.0.1localhost.localdomainlocalhost
::1localhost6.localdomain6localhost6
127.0.0.1namenodedatanode01

解压安装hadoop

[root@localhosthadoop]#tarzxvfhadoop-0.20.203.tar.gz
......
......
......
hadoop-0.20.203.0/src/contrib/ec2/bin/image/create-hadoop-image-remote
hadoop-0.20.203.0/src/contrib/ec2/bin/image/ec2-run-user-data
hadoop-0.20.203.0/src/contrib/ec2/bin/launch-hadoop-cluster
hadoop-0.20.203.0/src/contrib/ec2/bin/launch-hadoop-master
hadoop-0.20.203.0/src/contrib/ec2/bin/launch-hadoop-slaves
hadoop-0.20.203.0/src/contrib/ec2/bin/list-hadoop-clusters
hadoop-0.20.203.0/src/contrib/ec2/bin/terminate-hadoop-cluster
[root@localhosthadoop]#

进入hadoop配置conf

####################################
[root@localhostconf]#vihadoop-env.sh
#添加代码
#setjavaenvironment
exportJAVA_HOME=/usr/java/jdk1.6.0_26

#####################################
[root@localhostconf]#vicore-site.xml
<?xmlversion="1.0"?>
<?xml-stylesheettype="text/xsl"href="configuration.xsl"?>
<!--Putsite-specificpropertyoverridesinthisfile.-->
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://namenode:9000/</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/hadooptmp</value>
</property>
</configuration>

#######################################
[root@localhostconf]#vihdfs-site.xml
<?xmlversion="1.0"?>
<?xml-stylesheettype="text/xsl"href="configuration.xsl"?>
<!--Putsite-specificpropertyoverridesinthisfile.-->
<configuration>
<property>
<name>dfs.name.dir</name>
<value>/usr/local/hadoop/hdfs/name</value>
</property>
<property>
<name>dfs.data.dir</name>
<value>/usr/local/hadoop/hdfs/data</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>

#########################################
[root@localhostconf]#vimapred-site.xml
<?xmlversion="1.0"?>
<?xml-stylesheettype="text/xsl"href="configuration.xsl"?>
<!--Putsite-specificpropertyoverridesinthisfile.-->
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>namenode:9001</value>
</property>
<property>
<name>mapred.local.dir</name>
<value>/usr/local/hadoop/mapred/local</value>
</property>
<property>
<name>mapred.system.dir</name>
<value>/tmp/hadoop/mapred/system</value>
</property>
</configuration>

#########################################
[root@localhostconf]#vimasters
#localhost
namenode

#########################################
[root@localhostconf]#vislaves
#localhost
datanode01

启动 hadoop

#####################<spanstyle="font-size:small;">格式化namenode##############</span>



[root@localhostbin]#hadoopnamenode-format
11/06/2300:43:54INFOnamenode.NameNode:STARTUP_MSG:
/************************************************************
STARTUP_MSG:StartingNameNode
STARTUP_MSG:host=localhost.localdomain/127.0.0.1
STARTUP_MSG:args=[-format]
STARTUP_MSG:version=0.20.203.0
STARTUP_MSG:build=http://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-203-r1099333;compiledby'oom'onWedMay407:57:50PDT2011
************************************************************/
11/06/2300:43:55INFOutil.GSet:VMtype=32-bit
11/06/2300:43:55INFOutil.GSet:2%maxmemory=19.33375MB
11/06/2300:43:55INFOutil.GSet:capacity=2^22=4194304entries
11/06/2300:43:55INFOutil.GSet:recommended=4194304,actual=4194304
11/06/2300:43:56INFOnamenode.FSNamesystem:fsOwner=root
11/06/2300:43:56INFOnamenode.FSNamesystem:supergroup=supergroup
11/06/2300:43:56INFOnamenode.FSNamesystem:isPermissionEnabled=true
11/06/2300:43:56INFOnamenode.FSNamesystem:dfs.block.invalidate.limit=100
11/06/2300:43:56INFOnamenode.FSNamesystem:isAccessTokenEnabled=falseaccessKeyUpdateInterval=0min(s),accessTokenLifetime=0min(s)
11/06/2300:43:56INFOnamenode.NameNode:Cachingfilenamesoccuringmorethan10times
11/06/2300:43:57INFOcommon.Storage:Imagefileofsize110savedin0seconds.
11/06/2300:43:57INFOcommon.Storage:Storagedirectory/usr/local/hadoop/hdfs/namehasbeensuccessfullyformatted.
11/06/2300:43:57INFOnamenode.NameNode:SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG:ShuttingdownNameNodeatlocalhost.localdomain/127.0.0.1
************************************************************/
[root@localhostbin]#

###########################################
[root@localhostbin]#./start-all.sh
startingnamenode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-namenode-localhost.localdomain.out
datanode01:startingdatanode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-datanode-localhost.localdomain.out
namenode:startingsecondarynamenode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-secondarynamenode-localhost.localdomain.out
startingjobtracker,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-jobtracker-localhost.localdomain.out
datanode01:startingtasktracker,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-tasktracker-localhost.localdomain.out
[root@localhostbin]#jps
11971TaskTracker
11807SecondaryNameNode
11599NameNode
12022Jps
11710DataNode
11877JobTracker

查看集群状态

[root@localhostbin]#hadoopdfsadmin-report
ConfiguredCapacity:4055396352(3.78GB)
PresentCapacity:464142351(442.64MB)
DFSRemaining:464089088(442.59MB)
DFSUsed:53263(52.01KB)
DFSUsed%:0.01%
Underreplicatedblocks:0
Blockswithcorruptreplicas:0
Missingblocks:0

-------------------------------------------------
Datanodesavailable:1(1total,0dead)

Name:127.0.0.1:50010
DecommissionStatus:Normal
ConfiguredCapacity:4055396352(3.78GB)
DFSUsed:53263(52.01KB)
NonDFSUsed:3591254001(3.34GB)
DFSRemaining:464089088(442.59MB)
DFSUsed%:0%
DFSRemaining%:11.44%
Lastcontact:ThuJun2301:11:15PDT2011


[root@localhostbin]#

其他问题：1

####################启动报错##########
[root@localhostbin]#./start-all.sh
startingnamenode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-namenode-localhost.localdomain.out
Theauthenticityofhost'datanode01(127.0.0.1)'can'tbeestablished.
RSAkeyfingerprintis41:c8:d4:e4:60:71:6f:6a:33:6a:25:27:62:9b:e3:90.
Areyousureyouwanttocontinueconnecting(yes/no)?y
Pleasetype'yes'or'no':yes
datanode01:Warning:Permanentlyadded'datanode01'(RSA)tothelistofknownhosts.
datanode01:startingdatanode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-datanode-localhost.localdomain.out
<strong><spanstyle="color:#ff0000;">datanode01:Unrecognizedoption:-jvm
datanode01:CouldnotcreatetheJavavirtualmachine.</span>


</strong>



namenode:startingsecondarynamenode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-secondarynamenode-localhost.localdomain.out
startingjobtracker,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-jobtracker-localhost.localdomain.out
datanode01:startingtasktracker,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-tasktracker-localhost.localdomain.out
[root@localhostbin]#jps
10442JobTracker
10533TaskTracker
10386SecondaryNameNode
10201NameNode
10658Jps

################################################
[root@localhostbin]#vihadoop
elif["$COMMAND"="datanode"];then
CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
if[[$EUID-eq0]];then
HADOOP_OPTS="$HADOOP_OPTS-jvmserver$HADOOP_DATANODE_OPTS"
else
HADOOP_OPTS="$HADOOP_OPTS-server$HADOOP_DATANODE_OPTS"
fi

#http://javoft.net/2011/06/hadoop-unrecognized-option-jvm-could-not-create-the-java-virtual-machine/
#改为
elif["$COMMAND"="datanode"];then
CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
#if[[$EUID-eq0]];then
#HADOOP_OPTS="$HADOOP_OPTS-jvmserver$HADOOP_DATANODE_OPTS"
#else
HADOOP_OPTS="$HADOOP_OPTS-server$HADOOP_DATANODE_OPTS"
#fi

#或者换非root用户启动
#启动成功

2，启动时要关闭防火墙

查看运行情况：

http://localhost:50070

NameNode'localhost.localdomain:9000'
Started:ThuJun2301:07:18PDT2011
Version:0.20.203.0,r1099333
Compiled:WedMay407:57:50PDT2011byoom
Upgrades:Therearenoupgradesinprogress.

Browsethefilesystem
NamenodeLogs
ClusterSummary
6filesanddirectories,1blocks=7total.HeapSizeis31.38MB/966.69MB(3%)
ConfiguredCapacity:3.78GB
DFSUsed:52.01KB
NonDFSUsed:3.34GB
DFSRemaining:442.38MB
DFSUsed%:0%
DFSRemaining%:11.44%
LiveNodes:1
DeadNodes:0
DecommissioningNodes:0
NumberofUnder-ReplicatedBlocks:0

NameNodeStorage:
StorageDirectoryTypeState
/usr/local/hadoop/hdfs/nameIMAGE_AND_EDITSActive

http://localhost:50030

namenodeHadoopMap/ReduceAdministration
QuickLinks

*SchedulingInfo
*RunningJobs
*RetiredJobs
*LocalLogs

State:RUNNING
Started:ThuJun2301:07:30PDT2011
Version:0.20.203.0,r1099333
Compiled:WedMay407:57:50PDT2011byoom
Identifier:201106230107
ClusterSummary(HeapSizeis15.31MB/966.69MB)
RunningMapTasksRunningReduceTasksTotalSubmissionsNodesOccupiedMapSlotsOccupiedReduceSlotsReservedMapSlotsReservedReduceSlotsMapTaskCapacityReduceTaskCapacityAvg.Tasks/NodeBlacklistedNodesGraylistedNodesExcludedNodes
00010000224.00000

SchedulingInformation
QueueNameStateSchedulingInformation
defaultrunningN/A
Filter(Jobid,Priority,User,Name)
Example:'user:smith3200'willfilterby'smith'onlyintheuserfieldand'3200'inallfields
RunningJobs
none
RetiredJobs
none
LocalLogs
Logdirectory,JobTrackerHistoryThisisApacheHadooprelease0.20.203.0

测试：

##########建立目录名称##########
[root@localhostbin]#hadoopfs-mkdirtestFolder

###############拷贝文件到文件夹中
[root@localhostlocal]#ls
binetcgameshadoopincludeliblibexecsbinsharesrcSSH_key_file
[root@localhostlocal]#hadoopfs-copyFromLocalSSH_key_filetestFolder

进入web页面即可查看

参考：http://bxyzzy.blog.51cto.com/854497/352692

附： 准备FTP :yum install vsftpd (方便文件传输 和hadoop无关)

关闭防火墙：service iptables start

启动FTP：service vsftpd start