java操作hdfs

1  1 package hdfs;
2  2 
3  3 import java.io.FileNotFoundException;
4  4 import java.io.IOException;
5  5 import java.net.URI;
6  6 
7  7 import org.apache.hadoop.conf.Configuration;
8  8 import org.apache.hadoop.fs.FileStatus;
9  9 import org.apache.hadoop.fs.FileSystem;
10 10 import org.apache.hadoop.fs.LocatedFileStatus;
11 11 import org.apache.hadoop.fs.Path;
12 12 import org.apache.hadoop.fs.RemoteIterator;
13 13 import org.junit.Before;
14 14 import org.junit.Test;
15 15 
16 16 public class HdfsClientEasy {
17 17 
18 18     private FileSystem fs = null;
19 19 
20 20     @Before
21 21     public void getFs() throws Exception {
22 22 
23 23         // 拿到一个配置参数的封装对象，构造函数中就会对classpath下的xxx-site.xml文件进行解析
24 24         // 真实的项目工程中就应该把xxx-site.xml文件加入到工程中来
25 25         Configuration conf = new Configuration();
26 26         // to set a parameter, figure out the filesystem is hdfs
27 27         conf.set("fs.defaultFS", "hdfs://localhost:8000/");
28 28         conf.set("dfs.replication", "1");
29 29 
30 30         // 获取到一个具体文件系统的客户端实例对象，产生的实例究竟是哪一种文件系统的客户端，是根据conf中的相关参数来决定
31 31 //        fs = FileSystem.get(conf);
32 32         
33 33         //这种获取fs的方法可以指定访问hdfs的客户端身份
34 34         fs = FileSystem.get(new URI("hdfs://localhost:8000/"), conf, "wochu");
35 35 
36 36     }
37 37 
38 38     /**
39 39      * 上传文件
40 40      * 
41 41      * @throws IllegalArgumentException
42 42      * @throws IOException
43 43      */
44 44     @Test
45 45     public void testUpload() throws IllegalArgumentException, IOException {
46 46         fs.copyFromLocalFile(new Path("/Users/wochu/Desktop/tool/hdfs/test.txt"), new Path("/xue"));
47 47 
48 48     }
49 49 
50 50     /**
51 51      * 删除文件
52 52      * 
53 53      * @throws IllegalArgumentException
54 54      * @throws IOException
55 55      */
56 56     @Test
57 57     public void testRmfile() throws IllegalArgumentException, IOException {
58 58 
59 59         boolean res = fs.delete(new Path("/xue"), true);
60 60 
61 61         System.out.println(res ? "delete is successfully :)"
62 62                 : "it is failed :(");
63 63 
64 64     }
65 65 
66 66     /**
67 67      * 创建文件夹
68 68      * 
69 69      * @throws IllegalArgumentException
70 70      * @throws IOException
71 71      */
72 72     @Test
73 73     public void testMkdir() throws IllegalArgumentException, IOException {
74 74         fs.mkdirs(new Path("/xue"));
75 75 
76 76     }
77 77 
78 78     /**
79 79      * 重命名文件
80 80      * 
81 81      * @throws IllegalArgumentException
82 82      * @throws IOException
83 83      */
84 84     @Test
85 85     public void testRename() throws IllegalArgumentException, IOException {
86 86 
87 87         fs.rename(new Path("/xue/test.txt"), new Path("/xue/rename.txt"));
88 88 
89 89     }
90 90 
91 91     /**
92 92      * 列出目录下的文件信息
93 93      * 
94 94      * @throws FileNotFoundException
95 95      * @throws IllegalArgumentException
96 96      * @throws IOException
97 97      */
98 98     @Test
99 99     public void testListFiles() throws FileNotFoundException,
100100             IllegalArgumentException, IOException {
101101 
102102         // 递归列出文件
103103         RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(
104104                 new Path("/"), true);
105105 
106106         while (listFiles.hasNext()) {
107107 
108108             LocatedFileStatus file = listFiles.next();
109109 
110110             System.out.println(file.getPath().getName());
111111 
112112         }
113113 
114114         System.out.println("--------------------------------------------");
115115 
116116         // 列出文件及文件夹
117117         FileStatus[] status = fs.listStatus(new Path("/"));
118118         for (FileStatus file : status) {
119119 
120120             System.out.println(file.getPath().getName() + "   "
121121                     + (file.isDirectory() ? "d" : "f"));
122122 
123123         }
124124 
125125     }
126126 
127127     /**
128128      * 从hdfs中下载数据到本地
129129      * 
130130      * @throws IllegalArgumentException
131131      * @throws IOException
132132      */
133133     @Test
134134     public void testDownload() throws IllegalArgumentException, IOException {
135135 
136136         fs.copyToLocalFile(new Path("/xue/rename.txt"), new Path("/Users/wochu/Desktop/tool/hdfs/name.txt"));
137137 
138138     }
139139 
140140 }