姚斌分布式作业一

首先打开hadoop,

$ sbin/start-dfs.sh

$sbin/start-yarn.sh

执行一下jps 命令看一下.

ok,各任务都正常。

上次已经验证过了word count。注意每次生成都需要把上次的输出的文件夹删掉。

 

现在开始做hdfs的实验。

http://hadoop.apache.org/docs/r1.0.4/cn/hdfs_shell.html

http://blog.csdn.net/bigdatahappy/article/details/10068881

注意无需重新配置，新建的eclipce mapredruce project 只需要run on hadoop就可以了。

---------------------------------------------------------------------------------------------

作业一的要求是从一个文件动态增加的文件夹中读取文件，上传后删除。附上代码如下：

import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;

 
public class copyFile {
    //initialization
    static Configuration conf = new Configuration();
    static FileSystem hdfs;
    static FileSystem local;
    static {
        conf.set("fs.default.name","hdfs://localhost:9000");
        
        try {
            hdfs = FileSystem.get(conf);
            local = FileSystem.getLocal(conf);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) throws IOException {
    	//Path src = new Path("/home/byshen/HadoopInput/xjbz2.c");
    	//Path dst = new Path("/user/byshen/input");
    	Path inputDir = new Path("/home/byshen/视频");
    	Path acceptDir = new Path("/user/byshen/AcceptFile");
    	hdfs.mkdirs(acceptDir);
    	
    	
    	System.out.println("===== Mkdir ======\n");
    	
    	FileStatus files[] = local.listStatus(inputDir);
    	FSDataOutputStream out;
    	int i = 0;
    	while(files.length >0) {
    		System.out.println(files[i].getPath().getName());
    		FSDataInputStream in = local.open(files[i].getPath());
    		out = hdfs.create(new Path("/user/byshen/AcceptFile/" + files[i].getPath().getName()));
    		byte buffer[] = new byte[256];
    		int bytesRead = 0;
    		while( (bytesRead = in.read(buffer)) > 0) {
    			out.write(buffer,0,bytesRead);
    		}
    		out.close();
    		in.close();
    		
    		File toDelete = new File("/home/byshen/视频/" + files[i].getPath().getName());
    		
    		toDelete.delete();
    		if(toDelete.exists()) {
    			System.out.println("delete " + files[i].getPath().getName() + "failed");	
    		}
    		files = local.listStatus(inputDir);
    		System.out.println(files.length + " videos remained...");
    	}
    	System.out.println("all transfered success ... ");
    }
}