准备工作：IDEA > HDFS客户端准备

目录

文件上传

文件下载

文件夹删除

修改文件名称

查看文件详情

文件和文件夹的判断

完整代码

文件上传

注意conf.set("dfs.replication","2");的位置，位置不一样，设置的副本数不一样

HDFS参数优先级说明（由高到低）：代码中 > resources中的hdfs-site.xml > 集群中的设置cat /opt/module/hadoop-2.7.2/etc/hadoop/hdfs-site.xml  > 默认设置

在hdfs-site.xml中添加：

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration><property><name>dfs.replication</name><value>1</value></property>
</configuration>

//文件上传
@Test
public void testCopyFromLocalFile() throws IOException, URISyntaxException, InterruptedException {//1. 获取fs对象Configuration conf = new Configuration();conf.set("dfs.replication","2");FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");
//conf.set("dfs.replication","2");//2.执行上传APIfs.copyFromLocalFile(new Path("e:/copyFromLocalFile.txt"),new Path("/copyFromLocalFile_code.txt"));//3.关闭资源fs.close();
}

文件下载

查看copyToLocalFile源码

delSrc是否删除源数据

useRawLocalFileSystem（使用本地文件系统）默认值是False，在文件下载的时候会有crc校验

设置为true时，就没有crc

crc是一种安全校验，校验数据的可靠性，数据是否被改变了

@Test
//2. 文件下载
public void testCopyToLocalFile() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2. 执行下载操作
//    fs.copyToLocalFile(new Path("/user/zzz/input/wc.input"),new Path("e:/input.txt"));fs.copyToLocalFile(false,new Path("/user/zzz/input/wc.input"),new Path("e:/input1.txt"),true);//3.关闭资源fs.close();}

文件夹删除

fs.delete(new Path("/0529"),true);

两个参数最好同时在

也可以只有一个Path，但是不建议这么使用

@Test
//3. 文件删除
public void testDelete() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2. 执行删除操作//@Deprecated//不建议这么使用，但是可以使用
//    fs.delete(new Path("/0529"));fs.delete(new Path("/0529"),true);//3. 关闭资源fs.close();
}

修改文件名称

// 4. 文件更名
@Test
public void testRename() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2. 执行更名操作fs.rename(new Path("/panjinlian.txt"),new Path("/pjl.txt"));//3. 关闭资源fs.close();
}

查看文件详情

//5. 文件详情查看
@Test
public void testListFile() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2. 查看文件详情RemoteIterator<LocatedFileStatus> listFile = fs.listFiles(new Path("/"), true);while(listFile.hasNext()){LocatedFileStatus fileStatus = listFile.next();//查看文件名称、权限、长度、块信息System.out.println(fileStatus.getPath().getName());//文件名称System.out.println(fileStatus.getPermission());//文件权限System.out.println(fileStatus.getLen());//文件长度BlockLocation[] blockLocations = fileStatus.getBlockLocations();for(BlockLocation blockLocation: blockLocations){String[] hosts = blockLocation.getHosts();//副本具体存在于哪个主机上for(String host:hosts){System.out.println(host);}}System.out.println("---------------分割线---------------------");}//3. 关闭资源fs.close();
}

文件和文件夹的判断

//文件和文件夹判断
@Test
public void  testLineStatus() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"), conf, "zzz");//2. 具体判断操作FileStatus[] listStatus = fs.listStatus(new Path("/"));for (FileStatus fileStatus : listStatus) {if(fileStatus.isFile()){//文件System.out.println("f:" + fileStatus.getPath().getName());}else{//文件夹System.out.println("d:" + fileStatus.getPath().getName());}}//3. 关闭资源fs.close();
}

完整代码

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.Configuration;
import org.junit.Test;public class HDFSClient {public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {Configuration conf = new Configuration();//conf.set("fs.defaultFS","hdfs://hadoop131:9000");// 1 获取hdfs客户端对象FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");// 2 在hdfs上创建路径fs.mkdirs(new Path("/test/zzz"));// 3 关闭fs.close();System.out.println("over");}//文件上传@Testpublic void testCopyFromLocalFile() throws IOException, URISyntaxException, InterruptedException {//1. 获取fs对象Configuration conf = new Configuration();conf.set("dfs.replication","2");FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2.执行上传APIfs.copyFromLocalFile(new Path("e:/copyFromLocalFile.txt"),new Path("/copyFromLocalFile_code.txt"));//3.关闭资源fs.close();}@Test//2. 文件下载public void testCopyToLocalFile() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2. 执行下载操作//    fs.copyToLocalFile(new Path("/user/zzz/input/wc.input"),new Path("e:/input.txt"));fs.copyToLocalFile(false,new Path("/user/zzz/input/wc.input"),new Path("e:/input1.txt"),true);//3.关闭资源fs.close();}@Test//3. 文件删除public void testDelete() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2. 执行删除操作//@Deprecated//不建议这么使用，但是可以使用//    fs.delete(new Path("/0529"));fs.delete(new Path("/0529"),true);//3. 关闭资源fs.close();}// 4. 文件更名@Testpublic void testRename() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2. 执行更名操作fs.rename(new Path("/panjinlian.txt"),new Path("/pjl.txt"));//3. 关闭资源fs.close();}//5. 文件详情查看@Testpublic void testListFile() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"),conf,"zzz");//2. 查看文件详情RemoteIterator<LocatedFileStatus> listFile = fs.listFiles(new Path("/"), true);while(listFile.hasNext()){LocatedFileStatus fileStatus = listFile.next();//查看文件名称、权限、长度、块信息System.out.println(fileStatus.getPath().getName());//文件名称System.out.println(fileStatus.getPermission());//文件权限System.out.println(fileStatus.getLen());//文件长度BlockLocation[] blockLocations = fileStatus.getBlockLocations();for(BlockLocation blockLocation: blockLocations){String[] hosts = blockLocation.getHosts();//副本具体存在于哪个主机上for(String host:hosts){System.out.println(host);}}System.out.println("---------------分割线---------------------");}//3. 关闭资源fs.close();}//文件和文件夹判断@Testpublic void  testLineStatus() throws URISyntaxException, IOException, InterruptedException {//1. 获取对象Configuration conf = new Configuration();FileSystem fs = FileSystem.get(new URI("hdfs://192.168.223.101:9000"), conf, "zzz");//2. 具体判断操作FileStatus[] listStatus = fs.listStatus(new Path("/"));for (FileStatus fileStatus : listStatus) {if(fileStatus.isFile()){//文件System.out.println("f:" + fileStatus.getPath().getName());}else{//文件夹System.out.println("d:" + fileStatus.getPath().getName());}}//3. 关闭资源fs.close();}
}