Hadoop之HDFS04【JavaAPI操作】

 前面項目中我們是建立的java項目來示範的，但是hadoop相關的依賴太多了，不友善，本文通過maven項目來示範HDFS的java API操作

建立maven項目

相關的依賴

<dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.5.0</version>
    </dependency>

    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.5.1</version>
    </dependency>

    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.5.0</version>
    </dependency>

    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
    </dependency>
</dependencies>      

CRUD 操作

package com.sxt.test;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.Iterator;
import java.util.Map.Entry;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;


public class HdfsTest {

    FileSystem fs = null;
    
    Configuration conf = null;
    
    @Before
    public void test() throws Exception {
        // 1.加載配置檔案
        conf = new Configuration(true);
        // 2.擷取FileSystem對象
        fs = FileSystem.get(new URI("hdfs://hadoop-node01:9000"),conf,"root");
    }
    /**
     * 上傳檔案
     * @throws IOException 
     * @throws IllegalArgumentException 
     */
    @Test
    public void updateFile() throws IllegalArgumentException, IOException{
        fs.copyFromLocalFile(true, true, new Path("c:/unintall.log"), new Path("/"));
        fs.close();
    }
    
    /**
     * 下載下傳檔案
     * @throws Exception
     */
    @Test
    public void testDownload() throws Exception {
        fs.copyToLocalFile(new Path("/a.txt"), new Path("c:/tools"));
        fs.close();
    }
    
    
    /**
     * 列印參數
     */
    @Test
    public void testConf(){
        Iterator<Entry<String, String>> it = conf.iterator();
        while(it.hasNext()){
            Entry<String, String> ent = it.next();
            System.out.println(ent.getKey() + " : " + ent.getValue());
        }
    }

    /**
     * 建立檔案夾
     * @throws Exception
     */
    @Test
    public void testMkdir() throws Exception {
        boolean mkdirs = fs.mkdirs(new Path("/testmkdir/aaa/bbb"));
        System.out.println(mkdirs);
        
    }
    
    /**
     * 删除檔案夾
     * @throws Exception
     */
    @Test
    public void testDelete() throws Exception {
        // recursive 是否遞歸删除
        boolean flag = fs.delete(new Path("/testmkdir/aaa"), true);
        System.out.println(flag);
        
    }
    
    /**
     * 遞歸列出指定目錄下所有子檔案夾中的檔案
     * @throws Exception
     */
    @Test
    public void testLs() throws Exception {
        
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
        
        while(listFiles.hasNext()){
            LocatedFileStatus fileStatus = listFiles.next();
            System.out.println("blocksize: " +fileStatus.getBlockSize());
            System.out.println("owner: " +fileStatus.getOwner());
            System.out.println("Replication: " +fileStatus.getReplication());
            System.out.println("Permission: " +fileStatus.getPermission());
            System.out.println("Name: " +fileStatus.getPath().getName());
            System.out.println("------------------");
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for(BlockLocation b:blockLocations){
                System.out.println("塊起始偏移量: " +b.getOffset());
                System.out.println("塊長度:" + b.getLength());
                //塊所在的datanode節點
                String[] datanodes = b.getHosts();
                for(String dn:datanodes){
                System.out.println("datanode:" + dn);
                }
            }   
        }       
    }
    
    
    /**
     * 擷取檔案的類型
     * @throws Exception
     */
    @Test
    public void testLs2() throws Exception {        
        FileStatus[] listStatus = fs.listStatus(new Path("/"));
        for(FileStatus file :listStatus){
            System.out.println("name: " + file.getPath().getName());
            System.out.println((file.isFile()?"file":"directory"));
        }
    }
    /**
     * 通過流的方式将檔案上傳到HDFS系統中
     * @throws Exception
     * @throws IOException
     */
    @Test
    public void testStreamUpload() throws Exception, IOException{
        // 輸出流
        FSDataOutputStream out = fs.create(new Path("/dpb.txt"), true);
        // 位元組輸入流
        InputStream in = new FileInputStream("c:/tools/aaaaa.txt");
        IOUtils.copy(in, out);
        in.close();
        out.close();
    }
    
    /**
     * 通過流的方式擷取HDFS上的資料
     * @throws Exception 
     * @throws IllegalArgumentException 
     */
    @Test
    public void testStreamDownload() throws IllegalArgumentException, Exception{
        FSDataInputStream in = fs.open(new Path("/dpb.txt"));
        OutputStream out = new FileOutputStream("c:/tools/a1.txt");
        IOUtils.copy(in, out);
        out.close();
        in.close();
    }
    
    /**
     * 擷取HDFS中的檔案的一部分内容
     * @throws Exception 
     * @throws IllegalArgumentException 
     */
    @Test
    public void testRandomAccess() throws IllegalArgumentException, Exception{
        FSDataInputStream in = fs.open(new Path("/dpb.txt"));
        in.seek(4);
        OutputStream out = new FileOutputStream("c:/tools/a2.txt");
        IOUtils.copy(in, out);
        out.close();
        in.close();
    }
    
    /**
     * 顯示HDFS上檔案的内容
     * @throws IOException 
     * @throws IllegalArgumentException 
     */
    @Test
    public void testCat() throws IllegalArgumentException, IOException{
        FSDataInputStream in = fs.open(new Path("/dpb.txt"));
        // 将輸出的目的地執行控制台即可~
        IOUtils.copy(in, System.out);
    }
}      

隻放出最後一個方法的輸出。其他方法執行效果請各自嘗試