Hadoop配置文件，参数的优先级

参数优先级排序：

1. 客户端代码中设置的值
2. ClassPath下的用户自定义的配置文件    (project下的配置文件，例    如/root/IdeaProjects/hdfsClient/src/main/resources/hdfs-site.xml)）
3. 服务器的自定义配置文件（XXX-site.xml 路径为usr/local/hadoop/etc/hadoop）
4. 服务器的默认配置（XXX-default.xml 路径为cd usr/local/hadoop/etc/hadoop）

1.测试（2）和（3）的优先级

在项目中的resources中创建hdfs-site.xml文件，将如下内容拷贝进去（表明文件存储的份数为1份，而在/usr/local/hadoop/etc/hadoop/hdfs-site.xml中表明文件存储的份数为3份）

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

执行如下代码的testPut1：

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class hdfsClient {
    /**
     *
     */
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException,IOException,InterruptedException {
        URI uri=new URI("hdfs://172.18.0.2:9000");
        Configuration configuration=new Configuration();
        configuration.set("dfs.replication","2");
        System.out.println("dfs.replication:"+configuration.get("dfs.replication"));
        String user="root";
        fs=FileSystem.get(uri,configuration,user);
    }

    @After
    public void close() throws IOException {
        fs.close();
    }

    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/youxiuderen1"));
    }

    @Test
    public void testmkdir1() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/youxiuderen2"));
    }

    @Test
    public void testPut() throws URISyntaxException, IOException, InterruptedException {
        fs.copyFromLocalFile(false,false,new Path("/root/IdeaProjects/hdfsClient/jianglanshu.txt"), new Path("/youxiuderen1"));
    }

   
    @Test
    public void testput1() throws  IOException{
        fs.copyFromLocalFile(true,true,new Path("/root/IdeaProjects/hdfsClient/jianglanshu.txt"),new Path("/youxiuderen1"));
    }

    @Test
    public void testGet() throws IOException {
        fs.copyToLocalFile(false,new Path("/youxiuderen1/jianglanshu.txt"),new Path("/root/IdeaProjects/hdfsClient"),false);
    }

    @Test
    public void testGet1() throws IOException {
        fs.copyToLocalFile(false, new Path("/youxiuderen1/jianglanshu.txt"), new Path("/root/IdeaProjects/hdfsClient/jianglanshu1.txt"), true);
    }

    @Test
    public void testGet2() throws IOException {
        fs.copyToLocalFile(true, new Path("/youxiuderen1/jianglanshu.txt"), new Path("/root/IdeaProjects/hdfsClient/jianglanshu2.txt"), true);
    }

}

结果为：复制份数为1，所以在项目中的resources中创建hdfs-site.xml文件的优先级要高于服务器的自定义配置文件（XXX-site.xml路径为/usr/local/hadoop/hadoop）

2.测试（1）和（2）（3）的优先级顺序

在代码中添加configuration.set（"dfs.replication”，"2"）；设置份数为2份，在这之前设置了project 下的配置文件，Iroot/ldeaProjects/hdfsClient/src/main/resources/hdfs-site. xml份数为1份，服务器的自定义配置文件/usr/local/hadoop/etc/hadoop/hdfs-site.xml份数为3份

    @Before
    public void init() throws URISyntaxException,IOException,InterruptedException {
        URI uri=new URI("hdfs://172.18.0.2:9000");
        Configuration configuration=new Configuration();
        configuration.set("dfs.replication","2");
        System.out.println("dfs.replication:"+configuration.get("dfs.replication"));
        String user="root";
        fs=FileSystem.get(uri,configuration,user);
    }

    @After
    public void close() throws IOException {
        fs.close();

再去执行testPut1

（这里我把正确的testPut1漏掉了，重新加上之后执行出来显示文件不存在，因为已经删除，所以我又重新新建了一个文件）

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class hdfsClient {
    /**
     *
     */
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException,IOException,InterruptedException {
        URI uri=new URI("hdfs://172.18.0.2:9000");
        Configuration configuration=new Configuration();
        configuration.set("dfs.replication","2");
        System.out.println("dfs.replication:"+configuration.get("dfs.replication"));
        String user="root";
        fs=FileSystem.get(uri,configuration,user);
    }

    @After
    public void close() throws IOException {
        fs.close();
    }

    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/youxiuderen1"));
    }

    @Test
    public void testmkdir1() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/youxiuderen2"));
    }

    @Test
    public void testPut() throws URISyntaxException, IOException, InterruptedException {
        fs.copyFromLocalFile(false,false,new Path("/root/IdeaProjects/hdfsClient/jianglanshu.txt"), new Path("/youxiuderen1"));
    }

    @Test
    public void testput1() throws  IOException{
        fs.copyFromLocalFile(false,true,new Path("/root/IdeaProjects/hdfsClient/jianglanshu.txt"),new Path("/youxiuderen1"));
    }

    @Test
    public void testput2() throws  IOException{
        fs.copyFromLocalFile(true,true,new Path("/root/IdeaProjects/hdfsClient/jianglanshu.txt"),new Path("/youxiuderen1"));
    }

    @Test
    public void testGet() throws IOException {
        fs.copyToLocalFile(false,new Path("/youxiuderen1/jianglanshu.txt"),new Path("/root/IdeaProjects/hdfsClient"),false);
    }

    @Test
    public void testGet1() throws IOException {
        fs.copyToLocalFile(false, new Path("/youxiuderen1/jianglanshu.txt"), new Path("/root/IdeaProjects/hdfsClient/jianglanshu1.txt"), true);
    }

    @Test
    public void testGet2() throws IOException {
        fs.copyToLocalFile(true, new Path("/youxiuderen1/jianglanshu.txt"), new Path("/root/IdeaProjects/hdfsClient/jianglanshu2.txt"), true);
    }

}