摘要：搭建集群首先搭建環境版本大于安裝主機監控程序和加速器這里使用了的版本，因為安裝方便放開更改放開配置配置配置配置配置配置文件追加配置和在文件第一行中添加測試鏈接文件

首先搭建Docker環境,Docker版本大于1.3.2

安裝主機監控程序和加速器(curl -sSL https://get.daocloud.io/daomonit/install.sh | sh -s 7a029f60d36056fe1b85fabca6a133887245abe6)

docker pull daocloud.io/library/centos:centos6.7

docker run -it -h master --name master insaneworks/centos /bin/bash

yum install -y gcc vim openssh-server openssh-clients java-1.7.0-openjdk-devel.x86_64 tar wget

這里java使用了openjdk1.7.0的版本，因為安裝方便

vim /etc/ssh/sshd_config

放開PermitEmptyPasswords no
更改UsePAM no
放開PermitRootLogin yes

/etc/rc.d/init.d/sshd start

ssh-keygen -t rsa -P ""

cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

配置/etc/profile

export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk.x86_64/
export CLASSPATH=.:$JAVA_HOME/lib/dt:$JAVA_HOME/lib/tools.jar
export HADOOP_HOME=/root/hadoop-2.7.1
export PATH=$JAVA_HOME/bin:$PATH:$HADOOP_HOME/bin

11. 配置core-site.xml

    
        fs.defaultFS
        hdfs://master:9000
    
    
        fs.defaultFS
        hdfs://master:9000
    
    
        io.file.buffer.size
        131702
    
    
        hadoop.tmp.dir
        file:/root/hadoop-2.7.1/tmp
    

12. 配置hdfs-site.xml

    
        dfs.namenode.name.dir
        file:/root/hadoop-2.7.1/dfs/name
    
    
        dfs.datanode.data.dir
        file:/root/hadoop-2.7.1/dfs/data
    
    
        dfs.replication
        2
    
    
        dfs.namenode.secondary.http-address
        master:9001
    
    
        dfs.webhdfs.enabled
        true
    

13. 配置mapred-site.xml

    
        mapreduce.framework.name
        yarn
    
    
        mapreduce.jobhistory.address
        master:10020
    
    
        mapreduce.jobhistory.webapp.address
        master:19888
    

14. 配置yarn.site.xml

    
        yarn.nodemanager.aux-services
        mapreduce_shuffle
    
    
        yarn.nodemanager.auxservices.mapreduce.shuffle.class
        org.apache.hadoop.mapred.ShuffleHandler
    
    
        yarn.resourcemanager.address
        master:8032
    
    
        yarn.resourcemanager.scheduler.address
        master:8030
    
    
        yarn.resourcemanager.resource-tracker.address
        master:8031
    
    
        yarn.resourcemanager.admin.address
        master:8033
    
    
        yarn.resourcemanager.webapp.address
        master:8088
    
    
        yarn.nodemanager.resource.memory-mb
        1024
    

15. 配置etc/hadoop/slaves文件

追加
slave1
slave2
slave3

16. 配置hadoop-env.sh和yarn-env.sh

在文件第一行中添加：export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk.x86_64/

17. 測試鏈接文件

ldd /root/hadoop-2.7.1/lib/native/libhadoop.so.1.0.0
/root/hadoop-2.7.1/lib/native/libhadoop.so.1.0.0: /lib64/libc.so.6: version `GLIBC_2.14" not found (required by /root/hadoop-2.7.1/lib/native/libhadoop.so.1.0.0)
linux-vdso.so.1 => (0x00007fff24dbc000)
libdl.so.2 => /lib64/libdl.so.2 (0x00007ff8c6371000)
libc.so.6 => /lib64/libc.so.6 (0x00007ff8c5fdc000)
/lib64/ld-linux-x86-64.so.2 (0x00007ff8c679b000)

18. 安裝glibc-2.14

tar zxvf glibc-2.14.tar.gz
cd glibc-2.14
mkdir build
cd build
../configure --prefix=/usr/local/glibc-2.14
make
make install
ln -sf /usr/local/glibc-2.14/lib/libc-2.14.so /lib64/libc.so.6

19. 測試鏈接文件

ldd /root/hadoop-2.7.1/lib/native/libhadoop.so.1.0.0
linux-vdso.so.1 => (0x00007fff72b7c000)
libdl.so.2 => /lib64/libdl.so.2 (0x00007fb996ce9000)
libc.so.6 => /lib64/libc.so.6 (0x00007fb99695c000)
/lib64/ld-linux-x86-64.so.2 (0x00007fb997113000

20. 提交改鏡像即可

docker commit master ice/hadoop

21. 查看鏡像列表

docker images
REPOSITORY                              TAG                 IMAGE ID            CREATED             VIRTUAL SIZE
ice/hadoop              latest              385a97765871        7 hours ago         1.731 GB
daocloud.io/library/centos              centos6.7           3fba1048142f        11 days ago         190.6 MB
daocloud.io/daocloud/daocloud-toolset   latest              aa5dc2eecd4a        6 weeks ago         145.8 MB
daocloud.io/daocloud/daomonit           latest              ae375c157c27        7 weeks ago         149 MB

22. 干掉該配置鏡像

docker rm master

23. 啟動集群腳本

docker run --rm -it -p 50070:50070 -p 19888:19888 -p 8088:8088 -p 50030:50030 -h master --name master ice/hadoop /bin/bash
docker run --rm -it -h slave1 --name slave1 ice/hadoop /bin/bash
docker run --rm -it -h slave2 --name slave2 ice/hadoop /bin/bash
docker run --rm -it -h slave3 --name slave3 ice/hadoop /bin/bash

24. 配置環境變量和sshd服務

source /etc/profile
/etc/rc.d/init.d/sshd start

25. 查看各個節點的IP

docker inspect --format="{{.NetworkSettings.IPAddress}}" master
docker inspect --format="{{.NetworkSettings.IPAddress}}" slave1
docker inspect --format="{{.NetworkSettings.IPAddress}}" slave2
docker inspect --format="{{.NetworkSettings.IPAddress}}" slave3

26. 啟動Hadoop集群

hadoop namenode -format
/root/hadoop-2.7.1/sbin/start-dfs.sh
/root/hadoop-2.7.1/sbin/start-yarn.sh
注意??第一執行腳本時，需要確認一次

27. 關閉腳本

/root/hadoop-2.7.1/sbin/stop-dfs.sh
/root/hadoop-2.7.1/sbin/stop-yarn.sh

28. 測試命令

hadoop fs -mkdir /input
hadoop fs -ls /
hadoop fs -put /root/hadoop-2.7.1/etc/hadoop/* /input/
hadoop jar /root/hadoop-2.7.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar wordcount /input/ /output/wordcount/