hadoop config 详解

1.配置文件：
  1. -rw-r--r-- 1 hadoop  4436 Jun  2 17:47 capacity-scheduler.xml
  2. -rw-r--r-- 1 hadoop 61163 Jun  2 17:47 core-default.xml
  3. -rw-r--r-- 1 hadoop  1779 Jun  2 17:47 core-site.xml
  4. -rwxr-xr-x 1 hadoop  4328 Jun  2 17:47 hadoop-env.sh
  5. -rw-r--r-- 1 hadoop  9683 Jun  2 17:47 hadoop-policy.xml
  6. -rw-r--r-- 1 hadoop 81473 Jun  2 17:47 hdfs-default.xml
  7. -rw-r--r-- 1 hadoop  3305 Jun  2 17:47 hdfs-site.xml
  8. -rwxr-xr-x 1 hadoop  1449 Jun  2 17:47 httpfs-env.sh
  9. -rw-r--r-- 1 hadoop   620 Jun  2 17:47 httpfs-site.xml
  10. -rw-r--r-- 1 hadoop  3518 Jun  2 17:47 kms-acls.xml
  11. -rwxr-xr-x 1 hadoop  1527 Jun  2 17:47 kms-env.sh
  12. -rw-r--r-- 1 hadoop  5511 Jun  2 17:47 kms-site.xml
  13. -rw-r--r-- 1 hadoop 76016 Jun  2 17:47 mapred-default.xml
  14. -rwxr-xr-x 1 hadoop  1383 Jun  2 17:47 mapred-env.sh
  15. -rw-r--r-- 1 hadoop  2644 Jun  2 17:47 mapred-site.xml
  16. -rw-r--r-- 1 hadoop     6 Jun  2 17:47 masters
  17. -rw-r--r-- 1 hadoop    26 Jun  2 17:47 slaves
  18. -rw-r--r-- 1 hadoop 63142 Jun  2 17:47 yarn-default.xml
  19. -rwxr-xr-x 1 hadoop  4601 Jun  2 17:47 yarn-env.sh
  20. -rw-r--r-- 1 hadoop  4448 Jun  2 17:47 yarn-site.xml
2.core-site.xml

  2. <configuration>
  4. <!--- global properties -->
  5. <property>
  6.   <name>hadoop.tmp.dir</name>
  7.   <value>/xwtec/hadoop</value>
  8.   <description>A base for other temporary directories.</description>
  9. </property>
  11. <!-- i/o properties -->
  12. <property>
  13.   <name>io.file.buffer.size</name>
  14.   <value>131072</value>
  15.   <description>The size of buffer for use in sequence files.
  16.   The size of this buffer should probably be a multiple of hardware
  17.   page size (4096 on Intel x86), and it determines how much data is
  18.   buffered during read and write operations.</description>
  19. </property>
  21. <!-- file system properties -->
  22. <property>
  23.   <name>fs.defaultFS</name>
  24.   <value>hdfs://earth:9000</value>
  25.   <description>The name of the default file system.  A URI whose
  26.   scheme and authority determine the FileSystem implementation.  The
  27.   uri's scheme determines the config property (fs.SCHEME.impl) naming
  28.   the FileSystem implementation class.  The uri's authority is used to
  29.   determine the host, port, etc. for a filesystem.</description>
  30. </property>
  32. </configuration>

3.hdfs-site.xml
  1. <property>
  2.   <name>dfs.namenode.name.dir</name>
  3.   <value>file:///xwtec/hadoop/dfs/name</value>
  4.   <description>Determines where on the local filesystem the DFS name node
  5.       should store the name table(fsimage).  If this is a comma-delimited list
  6.       of directories then the name table is replicated in all of the
  7.       directories, for redundancy. </description>
  8. </property>
  10. <property>
  11.   <name>dfs.permissions.enabled</name>
  12.   <value>false</value>
  13.   <description>
  14.     If "true", enable permission checking in HDFS.
  15.     If "false", permission checking is turned off,
  16.     but all other behavior is unchanged.
  17.     Switching from one parameter value to the other does not change the mode,
  18.     owner or group of files or directories.
  19.   </description>
  20. </property>
  22. <property>
  23.   <name>dfs.datanode.data.dir</name>
  24.   <value>/xwtec/hadoop/dfs/data</value>
  25.   <description>Determines where on the local filesystem an DFS data node
  26.   should store its blocks.  If this is a comma-delimited
  27.   list of directories, then data will be stored in all named
  28.   directories, typically on different devices. The directories should be tagged
  29.   with corresponding storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS
  30.   storage policies. The default storage type will be DISK if the directory does
  31.   not have a storage type tagged explicitly. Directories that do not exist will
  32.   be created if local filesystem permission allows.
  33.   </description>
  35. <property>
  36.   <name>dfs.blocksize</name>
  37.   <value>134217728</value>
  38.   <description>
  39.       The default block size for new files, in bytes.
  40.       You can use the following suffix (case insensitive):
  41.       k(kilo), m(mega), g(giga), t(tera), p(peta), e(exa) to specify the size (such as 128k, 512m, 1g, etc.),
  42.       Or provide complete size in bytes (such as 134217728 for 128 MB).
  43.   </description>
  44. </property>
  45. </property>
  47. <property>
  48.   <name>dfs.namenode.secondary.http-address</name>
  49.   <value>mercury:50090</value>
  50.   <description>
  51.     The secondary namenode http server address and port.
  52.   </description>
  53. </property>
  55. <property>
  56.   <name>dfs.namenode.secondary.https-address</name>
  57.   <value>mercury:50091</value>
  58.   <description>
  59.     The secondary namenode HTTPS server address and port.
  60.   </description>
  61. </property>
  63. <property>
  64.   <name>dfs.replication</name>
  65.   <value>2</value>
  66.   <description>Default block replication. 
  67.   The actual number of replications can be specified when the file is created.
  68.   The default is used if replication is not specified in create time.
  69.   </description>
  70. </property>


4. yarn-site.xml
  1.   <!-- Resource Manager Configs -->
  2.   <property>
  3.     <name>yarn.resourcemanager.hostname</name>
  4.     <value>earth</value>
  5.     <description>The hostname of the RM.</description>
  6.   </property>    
  8.   <property>
  9.     <name>yarn.resourcemanager.scheduler.address</name>
  10.     <value>${yarn.resourcemanager.hostname}:8030</value>
  11.     <description>The address of the scheduler interface.</description>
  12.   </property>
  14.   <property>
  15.     <name>yarn.resourcemanager.resource-tracker.address</name>
  16.     <value>${yarn.resourcemanager.hostname}:8031</value>
  17.   </property>
  19.   <property>
  20.     <name>yarn.resourcemanager.address</name>
  21.     <value>${yarn.resourcemanager.hostname}:8032</value>
  22.     <description>The address of the applications manager interface in the RM.</description>
  23.   </property>
  25.   <property>
  26.     <name>yarn.resourcemanager.admin.address</name>
  27.     <value>${yarn.resourcemanager.hostname}:8033</value>
  28.     <description>The address of the RM admin interface.</description>
  29.   </property>
  31.   <property>
  32.     <name>yarn.resourcemanager.webapp.address</name>
  33.     <value>${yarn.resourcemanager.hostname}:8088</value>
  34.     <description>The http address of the RM web application.</description>
  35.   </property>
  37.   <property>
  38.     <name>yarn.resourcemanager.scheduler.class</name>
  39.     <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
  40.     <description>The class to use as the resource scheduler.</description>
  41.   </property>
  43.   <!-- Node Manager Configs -->
  44.   <property>
  45.     <name>yarn.nodemanager.hostname</name>
  46.     <value>0.0.0.0</value>
  47.     <description>The hostname of the NM.</description>
  48.   </property>
  50.   <property>
  51.     <name>yarn.nodemanager.local-dirs</name>
  52.     <value>${hadoop.tmp.dir}/nm-local-dir</value>
  53.     <description>List of directories to store localized files in. An 
  54.       application's localized file directory will be found in:
  55.       ${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}.
  56.       Individual containers' work directories, called container_${contid}, will
  57.       be subdirectories of this.
  58.    </description>
  59.   </property>
  61.   <property>
  62.     <!-- where's yarn.log.dir?-->
  63.     <name>yarn.nodemanager.log-dirs</name>
  64.     <value>${yarn.log.dir}/userlogs</value>
  65.     <description>
  66.       Where to store container logs. An application's localized log directory 
  67.       will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
  68.       Individual containers' log directories will be below this, in directories 
  69.       named container_{$contid}. Each container directory will contain the files
  70.       stderr, stdin, and syslog generated by that container.
  71.     </description>
  72.   </property>
  74.   <property>
  75.     <name>yarn.nodemanager.aux-services</name>
  76.     <value>mapreduce_shuffle</value>
  77.     <description>A comma separated list of services where service name should only
  78.       contain a-zA-Z0-9_ and can not start with numbers</description>
  79.   </property>
  81.   <property>
  82.     <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
  83.     <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  84.   </property>
  86.   <property>
  87.     <description>The minimum allocation for every container request at the RM,
  88.     in MBs. Memory requests lower than this will throw a
  89.     InvalidResourceRequestException.</description>
  90.     <name>yarn.scheduler.minimum-allocation-mb</name>
  91.     <value>1024</value>
  92.   </property>
  94.   <property>
  95.     <description>The maximum allocation for every container request at the RM,
  96.     in MBs. Memory requests higher than this will throw a
  97.     InvalidResourceRequestException.</description>
  98.     <name>yarn.scheduler.maximum-allocation-mb</name>
  99.     <value>6666</value>
  100.   </property>
  102.   <property>
  103.     <name>yarn.nodemanager.resource.memory-mb</name>
  104.     <value>7168</value>
  105.   </property>

5. mapred-site.xml
  1. <!-- MR YARN Application properties -->
  2. <property>
  3.   <name>mapreduce.framework.name</name>
  4.   <value>yarn</value>
  5.   <description>The runtime framework for executing MapReduce jobs.
  6.   Can be one of local, classic or yarn.
  7.   </description>
  8. </property>
  10. <!-- jobhistory properties -->
  11. <property>
  12.   <name>mapreduce.jobhistory.address</name>
  13.   <value>earth:10020</value>
  14.   <description>MapReduce JobHistory Server IPC host:port</description>
  15. </property>
  17. <property>
  18.   <name>mapreduce.jobhistory.webapp.address</name>
  19.   <value>earth:19888</value>
  20.   <description>MapReduce JobHistory Server Web UI host:port</description>
  21. </property>
  23. <property>
  24.   <name>mapreduce.jobhistory.intermediate-done-dir</name>
  25.   <value>${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate</value>
  26.   <description></description>
  27. </property>
  29. <property>
  30.   <name>mapreduce.jobhistory.done-dir</name>
  31.   <value>${yarn.app.mapreduce.am.staging-dir}/history/done</value>
  32.   <description></description>
  33. </property>
  35. <property>
  36.   <name>mapreduce.job.ubertask.enable</name>
  37.   <value>false</value>
  38.   <description>Whether to enable the small-jobs "ubertask" optimization,
  39.   which runs "sufficiently small" jobs sequentially within a single JVM.
  40.   "Small" is defined by the following maxmaps, maxreduces, and maxbytes
  41.   settings. Note that configurations for application masters also affect
  42.   the "Small" definition - yarn.app.mapreduce.am.resource.mb must be
  43.   larger than both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb,
  44.   and yarn.app.mapreduce.am.resource.cpu-vcores must be larger than
  45.   both mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores to enable
  46.   ubertask. Users may override this value.
  47.   </description>
  48. </property>
  50. <property>
  51.  <name>mapreduce.map.memory.mb</name>
  52.  <value>1024</value>
  53. </property>
  56. <property>
  57.   <name>mapreduce.reduce.memory.mb</name>
  58.   <value>1024</value>
  59. </property>

6.yarn-env.sh
  1. # default log directory & file
  2. if [ "$YARN_LOG_DIR" = "" ]; then
  3.   YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
  4. fi
  5. if [ "$YARN_LOGFILE" = "" ]; then
  6.   YARN_LOGFILE='yarn.log'
  7. fi
  9. # default policy file for service-level authorization
  10. if [ "$YARN_POLICYFILE" = "" ]; then
  11.   YARN_POLICYFILE="hadoop-policy.xml"
  12. fi
  14. # restore ordinary behaviour
  15. unset IFS
  18. YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
  19. YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
  20. YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
  21. YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
  22. YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
  23. YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
  24. YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
  25. YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
  26. if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
  27.   YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
  28. fi
  29. YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"

7.hadoop-env.xml
  1. # Set Hadoop-specific environment variables here.
  3. # The only required environment variable is JAVA_HOME.  All others are
  4. # optional.  When running a distributed configuration it is best to
  5. # set JAVA_HOME in this file, so that it is correctly defined on
  6. # remote nodes.
  8. # The java implementation to use.
  9. JAVA_HOME=/opt/jdk1.7.0_79
  10. export JAVA_HOME=${JAVA_HOME}
  12. # The jsvc implementation to use. Jsvc is required to run secure datanodes
  13. # that bind to privileged ports to provide authentication of data transfer
  14. # protocol.  Jsvc is not required if SASL is configured for authentication of
  15. # data transfer protocol using non-privileged ports.
  16. #export JSVC_HOME=${JSVC_HOME}
  18. export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}

  20. # Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
  21. for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
  22.   if [ "$HADOOP_CLASSPATH" ]; then
  23.     export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
  24.   else
  25.     export HADOOP_CLASSPATH=$f
  26.   fi
  27. done
  29. # The maximum amount of heap to use, in MB. Default is 1000.
  30. #export HADOOP_HEAPSIZE=
  31. #export HADOOP_NAMENODE_INIT_HEAPSIZE=""
  33. # Extra Java runtime options.  Empty by default.
  34. export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
  35. export HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=/opt/hadoop/lib/native"
  37. # Command specific options appended to HADOOP_OPTS when specified
  38. export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
  39. export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
  41. export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
  43. export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS"
  44. export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS"
  46. # The following applies to multiple commands (fs, dfs, fsck, distcp etc)
  47. export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
  48. #HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
  50. # On secure datanodes, user to run the datanode as after dropping privileges.
  51. # This **MUST** be uncommented to enable secure HDFS if using privileged ports
  52. # to provide authentication of data transfer protocol.  This **MUST NOT** be
  53. # defined if SASL is configured for authentication of data transfer protocol
  54. # using non-privileged ports.
  55. export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
  57. # Where log files are stored.  $HADOOP_HOME/logs by default.
  58. #export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
  60. # Where log files are stored in the secure data environment.
  61. export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
  63. ###
  64. # HDFS Mover specific parameters
  65. ###
  66. # Specify the JVM options to be used when starting the HDFS Mover.
  67. # These options will be appended to the options specified as HADOOP_OPTS
  68. # and therefore may override any similar flags set in HADOOP_OPTS
  69. #
  70. # export HADOOP_MOVER_OPTS=""
  72. ###
  73. # Advanced Users Only!
  74. ###
  76. # The directory where pid files are stored. /tmp by default.
  77. # NOTE: this should be set to a directory that can only be written to by 
  78. #       the user that will run the hadoop daemons.  Otherwise there is the
  79. #       potential for a symlink attack.
  80. export HADOOP_PID_DIR=${HADOOP_PID_DIR}
  81. export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
  83. # A string representing this instance of hadoop. $USER by default.
  84. export HADOOP_IDENT_STRING=$USER





posted @ 2016-07-07 15:33  iAthena  阅读(1118)  评论(0)    收藏  举报