三种环境下的hdfs运行

一、在centos7上运行

　·运行hdfs：首先创建一个用以运行hadoop程序的工作目录hadoop-test

在hadoop-env.sh里添加 export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/home/xu/hadoop-2.6.5/hadoop-test  

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class CatFile {
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(args[0]), conf);　　//"file:///E:/src.txt"   "hdfs://node:9000/test/WordCount/README.txt"
        FSDataInputStream in = null;
        try {
            in = fs.open(new Path(args[0]));　　//通过 open方法获得文件输入流
            in.seek(3); // go back to pos 3 of the file
            IOUtils.copyBytes(in, System.out, 4096, false);
            System.out.println(in.getPos());
            
        } finally {
        IOUtils.closeStream(in);
        }　　　　// fs.mkdirs(new Path("hdfs://node:9000/usr/hadoop/data0417"));
    }  　　　　　//  share/hadoop/common/hadoop-common-2.6.5.jar
}

编译 javac -cp .:/home/xu/hadoop-2.6.5/share/hadoop/common/hadoop-common-2.6.5.jar CatFile.java -d参数可用以指定.class文件生成路径

执行 hadoop HadoopTest /test/WordCount/README.txt 

　·运行WordCount：①容易出现错误【Could not find or load main class org.apache.hadoop.util.*】，

按链接所示再配置node主机文件/etc/profile和~/.bashrc文件并source启用就行，主要是hadoop的一些参数 ;

②在工作目录下创建一个文件夹jars并把commons-cli-1.2.jar、hadoop-common-2.6.5.jar、hadoop-annotations-2.6.5.jar、

hadoop-mapreduce-client-core-2.6.5.jar这四个文件复制过来，可以使用 find / -name file 查询 ； 

package oldtest;

import java.io.*;
import java.util.*;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.*;

public class WordCount extends Configured implements Tool {

   public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {

     static enum Counters { INPUT_WORDS }
    
     private final static IntWritable one = new IntWritable(1);
     private Text word = new Text();
    
     private boolean caseSensitive = true;
     private Set<String> patternsToSkip = new HashSet<String>();

     private long numRecords = 0;
     private String inputFile;

     public void configure(JobConf job) {
       caseSensitive = job.getBoolean("wordcount.case.sensitive", true);
       inputFile = job.get("map.input.file");

       if (job.getBoolean("wordcount.skip.patterns", false)) {
         Path[] patternsFiles = new Path[0];
         try {
               patternsFiles = DistributedCache.getLocalCacheFiles(job);
         } catch (IOException ioe) {
               System.err.println("Caught exception while getting cached files: " + StringUtils.stringifyException(ioe));
         }
         for (Path patternsFile : patternsFiles) {
               parseSkipFile(patternsFile);
             }
       }
         }

     private void parseSkipFile(Path patternsFile) {
       try {
        BufferedReader fis = new BufferedReader(new FileReader(patternsFile.toString()));
         String pattern = null;
         while ((pattern = fis.readLine()) != null) {
               patternsToSkip.add(pattern);
         }
       } catch (IOException ioe) {
         System.err.println("Caught exception while parsing the cached file '" + patternsFile + "' : " + StringUtils.stringifyException(ioe));
           }
         }

     public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
       String line = (caseSensitive) ? value.toString() : value.toString().toLowerCase();

           for (String pattern : patternsToSkip) {
         line = line.replaceAll(pattern, "");
           }

           StringTokenizer tokenizer = new StringTokenizer(line);
           while (tokenizer.hasMoreTokens()) {
             word.set(tokenizer.nextToken());
             output.collect(word, one);
             reporter.incrCounter(Counters.INPUT_WORDS, 1);
           }
    
           if ((++numRecords % 100) == 0) {
             reporter.setStatus("Finished processing " + numRecords + " records " + "from the input file: " + inputFile);
           }
         }
       }
    
       public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
         public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
           int sum = 0;
           while (values.hasNext()) {
             sum += values.next().get();
           }
           output.collect(key, new IntWritable(sum));
         }
       }
    
       public int run(String[] args) throws Exception {
         JobConf conf = new JobConf(getConf(), WordCount.class);
         conf.setJobName("wordcount");
    
         conf.setOutputKeyClass(Text.class);
         conf.setOutputValueClass(IntWritable.class);
    
         conf.setMapperClass(Map.class);
         conf.setCombinerClass(Reduce.class);
         conf.setReducerClass(Reduce.class);
    
         conf.setInputFormat(TextInputFormat.class);
         conf.setOutputFormat(TextOutputFormat.class);
    
         List<String> other_args = new ArrayList<String>();
         for (int i=0; i < args.length; ++i) {
           if ("-skip".equals(args[i])) {
             DistributedCache.addCacheFile(new Path(args[++i]).toUri(), conf);
             conf.setBoolean("wordcount.skip.patterns", true);
           } else {
             other_args.add(args[i]);
           }
         }
    
         FileInputFormat.setInputPaths(conf, new Path(other_args.get(0)));
         FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));

         JobClient.runJob(conf);
         return 0;
       }
    
       public static void main(String[] args) throws Exception {
         int res = ToolRunner.run(new Configuration(), new WordCount(), args);
         System.exit(res);
       }
    }

    

javac -cp .:/home/xu/hadoop-2.6.5/hadoop-test/jars/* -d ./classes/  WordCount.java
jar -cvf WordCount.jar ./classes/WordCount*.class
hadoop jar WordCount.jar WordCount /test/WordCount /result/WordCount0 

二、在windows下安装配置　　参考windows+hadoop配置 ubuntu18

　·jdk-8u131-windows-x64的下载安装和配置环境变量，注意：jdk和jre安装目录不同，路径上没有@#之类的字符。

　·hadoop-2.6.5.tar.gz的下载解压和配置环境变量(三个)，注意：需要下载hadoop-common-bin-master再替换掉hadoop-2.6.5的bin；

记得给hadoop-2.6.5添加EveryOne的完全控制权限，如果是要windows下搭建hadoop记得配置三个sh和四个xml文件。

　·eclipse-jee下载解压(一切设置默认)；①下载hadoop-eclipse-plugin-2.6.5并放入eclipse/plugins目录下；

②打开eclipse，window->Preference->左侧Hadoop Map/Reduce右侧browse选择Hadoop的安装目录；

③点击，好像Advanced parameters的设置不必要。  

三、eclipse上运行

　·New->hadoop project，为WordCount添加命令行参数：Run->Run Configration->arguement(多个参数以空格隔开)

 "hdfs://node:9000/test/WordCount" "hdfs://node:9000/result/WordCount0" ;运行 右击->Run as->on hadoop

最后使用HDFS的JAVA API

import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;

public class PutMerge {  　　　　//本地某目录下文件的合并上传至云端
    public static void main(String[] args) throws IOException {
        Configuration conf = new Configuration();
        byte buffer[] = new byte[256];  int n = 0;        
        try {  
            FileSystem fs  = FileSystem.get(new URI("args[0]"),conf);
            FSDataOutputStream out = fs.create(new Path("/usr/hadoop/Merge.txt"));  　　//云端out
            
            FileSystem fsLocal = FileSystem.getLocal(conf);  
            FileStatus[] inDir = fsLocal.listStatus(new Path(args[1]));  //本地in   
            for (int i=0; i<inDir.length; i++) {                
                FSDataInputStream in = fsLocal.open(inDir[i].getPath()); 
                
                while(( n = in.read(buffer)) > 0)   
                    out.write(buffer, 0, n );  
                in.close();
            }
            out.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
} 

①首先判定文件系统FileSystem  fs=FileSystem.get(URI.create(uri),  new Configuration())、fsLocal = FileSystem.getLocal(conf); 

②然后定位文件路径FSDataOutputStream out = fs.create(new Path(uri))、FSDataInputStream in = fsLocal.open(New Path(uri))

参考 ubuntu+hadoop 连 windows+eclipse

打包成bat文件右击管理员模式执行 start cmd /k "cd/d E:/applications/hadoop-2.6.5/sbin&&start-all" 

2020-06-07　17:18:23