环境：mac/linux

hadoop版本：3.1.1

安装特性：非HA

 

准备：

1. jdk8以上

2. ssh

3. 下载hadoop安装包

 

配置文件，这里都只有简易配置：

1. core-site.xml

         
              
       
        hadoop.tmp.dir
               
       
        file:/opt/hadoop-3.1.1/tmp
               
       
        A base for other temporary directories.
           
          
              
       
        io.file.buffer.size
               
       
        131072
           
          
              
       
        fs.defaultFS
               
       
        hdfs://HxaMac:9000
           
      
     

2. hdfs-site.xml

         
              
       
        dfs.namenode.http-address
               
       
        0.0.0.0:50070
           
          
              
       
        dfs.replication
               
       
        1
           
          
              
       
        dfs.namenode.name.dir
               
       
        file:/Users/hadoop/hdfs/name
           
          
              
       
        dfs.datanode.data.dir
               
       
        file:/Users/hadoop/hdfs/data
           
          
              
       
        dfs.webhdfs.enabled
               
       
        true
           
          
              
       
        dfs.permissions
               
       
        false
           
      
     

3. yarn-site.xml

         
              
       
        yarn.resourcemanager.address
               
       
        HxaMac:18040
           
          
              
       
        yarn.resourcemanager.scheduler.address
               
       
        HxaMac:18030
           
          
      
               
       
        yarn.resourcemanager.webapp.address
               
       
        0.0.0.0:8088
                
           
      
                  
       
        yarn.resourcemanager.resource-tracker.address
                  
       
        HxaMac:18025
                
           
      
                  
       
        yarn.resourcemanager.admin.address
                  
       
        HxaMac:18141
                
           
      
                   
       
        yarn.nodemanager.aux-services
                   
       
        mapreduce_shuffle
                
           
      
                    
       
        yarn.nodemanager.auxservices.mapreduce.shuffle.class
                    
       
        org.apache.hadoop.mapred.ShuffleHandler
                
          
              
       
        yarn.log-aggregation-enable
               
       
        true
           
          
              
       
        yarn.log-aggregation.retain-seconds
               
       
        864000
           
          
              
       
        yarn.log-aggregation.retain-check-interval-seconds
               
       
        -1
           
      
     

4. mapred-site.xml

         
              
       
        mapreduce.framework.name
               
       
        yarn
           
          
              
       
        mapreduce.application.classpath
               
       
                    /opt/hadoop-3.1.1/etc/hadoop,            /opt/hadoop-3.1.1/share/hadoop/common/*,            /opt/hadoop-3.1.1/share/hadoop/common/lib/*,            /opt/hadoop-3.1.1/share/hadoop/hdfs/*,            /opt/hadoop-3.1.1/share/hadoop/hdfs/lib/*,            /opt/hadoop-3.1.1/share/hadoop/mapreduce/*,            /opt/hadoop-3.1.1/share/hadoop/mapreduce/lib/*,            /opt/hadoop-3.1.1/share/hadoop/yarn/*,            /opt/hadoop-3.1.1/share/hadoop/yarn/lib/*        
           
          
              
       
        mapreduce.jobhistory.max-age-ms
               
       
        5184000000
           
      
     

5. hadoop-env.sh

export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_201.jdk/Contents/Homeexport HADOOP_OPTS=-Djava.net.preferIPv4Stack=true

6. yarn-env.sh

一般不动。

7. worker

HxaMac

8.yarn-worker

HxaMac

 

步骤：

1. 完成jdk8、ssh的准备

2. 解压hadoop，一般在/opt目录下

3. 修改环境变量，如/etc/bash.bashrc

JAVA_HOME=jdk_dirCLASSPATH=$JAVA_HOME/lib/PATH=$JAVA_HOME/bin:$PATHexport PATH JAVA_HOME CLASSPATHalias hput='hadoop fs -put'alias hget='hadoop fs -get'alias hls='hadoop fs -ls'alias hrm='hadoop fs -rm -r'alias hcat='hadoop fs -cat'HADOOP_INSTALL=/opt/hadoop-3.1.1PATH=$HADOOP_INSTALL/bin:$PATHPATH=$HADOOP_INSTALL/sbin:$PATHexport HADOOP_HOME=$HADOOP_INSTALLexport HADOOP_COMMON_HOME=$HADOOP_HOMEexport HADOOP_HDFS_HOME=$HADOOP_HOMEexport HADOOP_MAPRED_HOME=$HADOOP_HOMEexport HADOOP_YARN_HOME=$HADOOP_HOMEexport HADOOP_INSTALL=$HADOOP_HOMEexport HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/nativeexport HADOOP_LIBEXEC_DIR=$HADOOP_HOME/libexecexport JAVA_LIBRARY_PATH=$HADOOP_HOME/lib/native:$JAVA_LIBRARY_PATHexport HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoopexport HDFS_DATANODE_USER=hadoopexport HDFS_DATANODE_SECURE_USER=hadoopexport HDFS_SECONDARYNAMENODE_USER=hadoopexport HDFS_NAMENODE_USER=hadoop

4. hdfs namenode -format

5. start-dfs.sh

查看http://hxamac:50070/

6. start-yarn.sh

查看http://hxamac:8088

7. 测试hdfs: put一个小文件

hadoop fs -mkdir -p /user/hxa/hadoop fs -put test.txt /user/hxa/

8. 测试mapreduce任务

hadoop jar hadoop-mapreduce-examples-3.1.1.jar pi 10 10