Hadoop HDFS API操作-----附案例

通过FileSystem API访问读取HDFS数据

@Test

public void readByFS() throws IOException{

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(conf);

String _path = "hdfs://master1:9000/spaceQuota/text.txt";

FSDataInputStream  in = fs.open(new Path(_path));

IOUtils.copyBytes(in, System.out, 4096, true);

}
           

通过FileSystem API创建文件夹

@Test

public void mkdirByAPI()throws Exception{

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(conf);

Path _path = new Path("/mkdir_byAPI");

fs.mkdirs(_path);

fs.close();

}
           

FileStatus对象，获取文件的相关属性

@Test

public void getPar()throws Exception{

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(conf);

FileStatus fstu = fs.getFileStatus(new Path("/spaceQuota/text.txt"));

long _blocksize = fstu.getBlockSize();

System.out.println("getAccessTime="+fstu.getAccessTime());

System.out.println("blockszie="+_blocksize);

System.out.println("getGroup="+fstu.getGroup());

System.out.println("getLen="+fstu.getLen());

System.out.println("getModificationTime="+fstu.getModificationTime());

System.out.println("getOwner="+fstu.getOwner());

System.out.println("getReplication="+fstu.getReplication());

System.out.println("getPath="+fstu.getPath());

}
           

通过FileSystem API做write操作

@Test

public void  writeByAPI() throws IOException{

Configuration conf = new Configuration();

FileSystem fs  =FileSystem.get(conf);

Path file = new Path("/spaceQuota/hello.txt");

FSDataOutputStream out = fs.create(file);

out.write("hello world".getBytes());

out.close();

}

           

通过FileSystem API做写操作,动态设置相关参数:replication为2和blocksize为10字节

@Test

public void writeByAPIForBlocksize() throws IOException{

Configuration conf = new Configuration();

conf.set("fs.defaultFS", "hdfs://master1:9000");

conf.set("dfs.bytes-per-checksum", "10");

FileSystem fs  =FileSystem.get(conf);

Path file = new Path("/spaceQuota/hello6666.txt");

FSDataOutputStream out = fs.create(file, true, 4096,(short)2, 10);

out.write("hello world".getBytes());

out.close();

}

           

seek操作

@Test

public void seekByAPI() throws IOException{

Configuration conf  =new Configuration();

FileSystem fs = FileSystem.get(conf);

Path file = new Path("/spaceQuota/hello3333.txt");

FSDataInputStream in = fs.open(file);

IOUtils.copyBytes(in, System.out, 4096,false);//注意：stream不能关闭

in.seek(0);

IOUtils.copyBytes(in, System.out, 4096,true);

}

           

测试一致模型

@Test

public void  writeByAPIText() throws IOException{

Configuration conf = new Configuration();

conf.set("dfs.bytes-per-checksum", "10");

FileSystem fs  =FileSystem.get(conf);

Path file = new Path("/spaceQuota/hello.txt");

//创建文件时，文件立即可见

FSDataOutputStream out = fs.create(file,true,4096,(short)2,10);

out.write("hello world1a".getBytes());

out.flush();

out.write("hello world2a".getBytes());

out.hflush();

out.write("hello world3a".getBytes());

out.hsync();

out.close();

}
           

append追加操作

@Test

public void  appendByAPI() throws IOException{

Configuration conf = new Configuration();

conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");    //修改属性参数

FileSystem fs = FileSystem.get(conf);

Path file = new Path("/spaceQuota/hello.txt");

FSDataOutputStream out = fs.append(file);

out.writeChars("aaaa");

out.close();

}