[转载]hdfs c/c++ API
原文链接：http://blog.csdn.net/sprintfwater/article/details/8996214
1.建立、关闭与HDFS连接：hdfsConnect()、hdfsConnectAsUser()、hdfsDisconnect()。hdfsConnect()实际上是直接调用hdfsConnectAsUser。
2.打开、关闭HDFS文件：hdfsOpenFile()、hdfsCloseFile()。当用hdfsOpenFile()创建文件时，可以指定replication和blocksize参数。写打开一个文件时，隐含O_TRUNC标志，文件会被截断，写入是从文件头开始的。
3.读HDFS文件：hdfsRead()、hdfsPread()。两个函数都有可能返回少于用户要求的字节数，此时可以再次调用这两个函数读入剩下的部分（类似APUE中的readn实现）；只有在两个函数返回零时，我们才能断定到了文件末尾。
4.写HDFS文件：hdfsWrite()。HDFS不支持随机写，只能是从文件头顺序写入。
5.查询HDFS文件信息：hdfsGetPathInfo()
6.查询和设置HDFS文件读写偏移量：hdfsSeek()、hdfsTell()
7.查询数据块所在节点信息：hdfsGetHosts()。返回一个或多个数据块所在数据节点的信息，一个数据块可能存在多个数据节点上。
8.libhdfs中的函数是通过jni调用JAVA虚拟机，在虚拟机中构造对应的HDFS的JAVA类，然后反射调用该类的功能函数。总会发生JVM和程序之间内存拷贝的动作，性能方面值得注意。
9.HDFS不支持多个客户端同时写入的操作，无文件或是记录锁的概念。
10.建议只有超大文件才应该考虑放在HDFS上，而且最好对文件的访问是写一次，读多次。小文件不应该考虑放在HDFS上，得不偿失！
1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19 #ifndef LIBHDFS_HDFS_H 20 #define LIBHDFS_HDFS_H 21 22 #include <sys/types.h> 23 #include <sys/stat.h> 24 25 #include <fcntl.h> 26 #include <stdio.h> 27 #include <stdint.h> 28 #include <string.h> 29 #include <stdlib.h> 30 #include <time.h> 31 #include <errno.h> 32 33 #include <jni.h> 34 35 #ifndef O_RDONLY 36 #define O_RDONLY 1 37 #endif 38 39 #ifndef O_WRONLY 40 #define O_WRONLY 2 41 #endif 42 43 #ifndef EINTERNAL 44 #define EINTERNAL 255 45 #endif 46 47 48 /** All APIs set errno to meaningful values */ 49 #ifdef __cplusplus 50 extern "C" { 51 #endif 52 53 /** 54 * Some utility decls used in libhdfs. 55 */ 56 57 typedef int32_t tSize; /// size of data for read/write io ops 58 typedef time_t tTime; /// time type 59 typedef int64_t tOffset;/// offset within the file 60 typedef uint16_t tPort; /// port 61 typedef enum tObjectKind { 62 kObjectKindFile = 'F', 63 kObjectKindDirectory = 'D', 64 } tObjectKind; 65 66 67 /** 68 * The C reflection of org.apache.org.hadoop.FileSystem . 69 */ 70 typedef void* hdfsFS; 71 72 73 /** 74 * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . 75 */ 76 enum hdfsStreamType 77 { 78 UNINITIALIZED = 0, 79 INPUT = 1, 80 OUTPUT = 2, 81 }; 82 83 84 /** 85 * The 'file-handle' to a file in hdfs. 86 */ 87 struct hdfsFile_internal { 88 void* file; 89 enum hdfsStreamType type; 90 }; 91 typedef struct hdfsFile_internal* hdfsFile; 92 93 94 /** 95 * hdfsConnect - Connect to a hdfs file system. 96 * Connect to the hdfs. 97 * @param host A string containing either a host name, or an ip address 98 * of the namenode of a hdfs cluster. 'host' should be passed as NULL if 99 * you want to connect to local filesystem. 'host' should be passed as 100 * 'default' (and port as 0) to used the 'configured' filesystem 101 * (hadoop-site/hadoop-default.xml). 102 * @param port The port on which the server is listening. 103 * @return Returns a handle to the filesystem or NULL on error. 104 */ 105 hdfsFS hdfsConnect(const char* host, tPort port); 106 107 108 /** 109 * hdfsDisconnect - Disconnect from the hdfs file system. 110 * Disconnect from hdfs. 111 * @param fs The configured filesystem handle. 112 * @return Returns 0 on success, -1 on error. 113 */ 114 int hdfsDisconnect(hdfsFS fs); 115 116 117 /** 118 * hdfsOpenFile - Open a hdfs file in given mode. 119 * @param fs The configured filesystem handle. 120 * @param path The full path to the file. 121 * @param flags Either O_RDONLY or O_WRONLY, for read-only or write-only. 122 * @param bufferSize Size of buffer for read/write - pass 0 if you want 123 * to use the default configured values. 124 * @param replication Block replication - pass 0 if you want to use 125 * the default configured values. 126 * @param blocksize Size of block - pass 0 if you want to use the 127 * default configured values. 128 * @return Returns the handle to the open file or NULL on error. 129 */ 130 hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, 131 int bufferSize, short replication, tSize blocksize); 132 133 134 /** 135 * hdfsCloseFile - Close an open file. 136 * @param fs The configured filesystem handle. 137 * @param file The file handle. 138 * @return Returns 0 on success, -1 on error. 139 */ 140 int hdfsCloseFile(hdfsFS fs, hdfsFile file); 141 142 143 /** 144 * hdfsExists - Checks if a given path exsits on the filesystem 145 * @param fs The configured filesystem handle. 146 * @param path The path to look for 147 * @return Returns 0 on success, -1 on error. 148 */ 149 int hdfsExists(hdfsFS fs, const char *path); 150 151 152 /** 153 * hdfsSeek - Seek to given offset in file. 154 * This works only for files opened in read-only mode. 155 * @param fs The configured filesystem handle. 156 * @param file The file handle. 157 * @param desiredPos Offset into the file to seek into. 158 * @return Returns 0 on success, -1 on error. 159 */ 160 int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); 161 162 163 /** 164 * hdfsTell - Get the current offset in the file, in bytes. 165 * @param fs The configured filesystem handle. 166 * @param file The file handle. 167 * @return Current offset, -1 on error. 168 */ 169 tOffset hdfsTell(hdfsFS fs, hdfsFile file); 170 171 172 /** 173 * hdfsRead - Read data from an open file. 174 * @param fs The configured filesystem handle. 175 * @param file The file handle. 176 * @param buffer The buffer to copy read bytes into. 177 * @param length The length of the buffer. 178 * @return Returns the number of bytes actually read, possibly less 179 * than than length;-1 on error. 180 */ 181 tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length); 182 183 184 /** 185 * hdfsPread - Positional read of data from an open file. 186 * @param fs The configured filesystem handle. 187 * @param file The file handle. 188 * @param position Position from which to read 189 * @param buffer The buffer to copy read bytes into. 190 * @param length The length of the buffer. 191 * @return Returns the number of bytes actually read, possibly less than 192 * than length;-1 on error. 193 */ 194 tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, 195 void* buffer, tSize length); 196 197 198 /** 199 * hdfsWrite - Write data into an open file. 200 * @param fs The configured filesystem handle. 201 * @param file The file handle. 202 * @param buffer The data. 203 * @param length The no. of bytes to write. 204 * @return Returns the number of bytes written, -1 on error. 205 */ 206 tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer, 207 tSize length); 208 209 210 /** 211 * hdfsWrite - Flush the data. 212 * @param fs The configured filesystem handle. 213 * @param file The file handle. 214 * @return Returns 0 on success, -1 on error. 215 */ 216 int hdfsFlush(hdfsFS fs, hdfsFile file); 217 218 219 /** 220 * hdfsAvailable - Number of bytes that can be read from this 221 * input stream without blocking. 222 * @param fs The configured filesystem handle. 223 * @param file The file handle. 224 * @return Returns available bytes; -1 on error. 225 */ 226 int hdfsAvailable(hdfsFS fs, hdfsFile file); 227 228 229 /** 230 * hdfsCopy - Copy file from one filesystem to another. 231 * @param srcFS The handle to source filesystem. 232 * @param src The path of source file. 233 * @param dstFS The handle to destination filesystem. 234 * @param dst The path of destination file. 235 * @return Returns 0 on success, -1 on error. 236 */ 237 int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); 238 239 240 /** 241 * hdfsMove - Move file from one filesystem to another. 242 * @param srcFS The handle to source filesystem. 243 * @param src The path of source file. 244 * @param dstFS The handle to destination filesystem. 245 * @param dst The path of destination file. 246 * @return Returns 0 on success, -1 on error. 247 */ 248 int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); 249 250 251 /** 252 * hdfsDelete - Delete file. 253 * @param fs The configured filesystem handle. 254 * @param path The path of the file. 255 * @return Returns 0 on success, -1 on error. 256 */ 257 int hdfsDelete(hdfsFS fs, const char* path); 258 259 260 /** 261 * hdfsRename - Rename file. 262 * @param fs The configured filesystem handle. 263 * @param oldPath The path of the source file. 264 * @param newPath The path of the destination file. 265 * @return Returns 0 on success, -1 on error. 266 */ 267 int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath); 268 269 270 /** 271 * hdfsGetWorkingDirectory - Get the current working directory for 272 * the given filesystem. 273 * @param fs The configured filesystem handle. 274 * @param buffer The user-buffer to copy path of cwd into. 275 * @param bufferSize The length of user-buffer. 276 * @return Returns buffer, NULL on error. 277 */ 278 char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize); 279 280 281 /** 282 * hdfsSetWorkingDirectory - Set the working directory. All relative 283 * paths will be resolved relative to it. 284 * @param fs The configured filesystem handle. 285 * @param path The path of the new 'cwd'. 286 * @return Returns 0 on success, -1 on error. 287 */ 288 int hdfsSetWorkingDirectory(hdfsFS fs, const char* path); 289 290 291 /** 292 * hdfsCreateDirectory - Make the given file and all non-existent 293 * parents into directories. 294 * @param fs The configured filesystem handle. 295 * @param path The path of the directory. 296 * @return Returns 0 on success, -1 on error. 297 */ 298 int hdfsCreateDirectory(hdfsFS fs, const char* path); 299 300 301 /** 302 * hdfsSetReplication - Set the replication of the specified 303 * file to the supplied value 304 * @param fs The configured filesystem handle. 305 * @param path The path of the file. 306 * @return Returns 0 on success, -1 on error. 307 */ 308 int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication); 309 310 311 /** 312 * hdfsFileInfo - Information about a file/directory. 313 */ 314 typedef struct { 315 tObjectKind mKind; /* file or directory */ 316 char *mName; /* the name of the file */ 317 tTime mLastMod; /* the last modification time for the file*/ 318 tOffset mSize; /* the size of the file in bytes */ 319 short mReplication; /* the count of replicas */ 320 tOffset mBlockSize; /* the block size for the file */ 321 } hdfsFileInfo; 322 323 324 /** 325 * hdfsListDirectory - Get list of files/directories for a given 326 * directory-path. hdfsFreeFileInfo should be called to deallocate memory. 327 * @param fs The configured filesystem handle. 328 * @param path The path of the directory. 329 * @param numEntries Set to the number of files/directories in path. 330 * @return Returns a dynamically-allocated array of hdfsFileInfo 331 * objects; NULL on error. 332 */ 333 hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, 334 int *numEntries); 335 336 337 /** 338 * hdfsGetPathInfo - Get information about a path as a (dynamically 339 * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be 340 * called when the pointer is no longer needed. 341 * @param fs The configured filesystem handle. 342 * @param path The path of the file. 343 * @return Returns a dynamically-allocated hdfsFileInfo object; 344 * NULL on error. 345 */ 346 hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path); 347 348 349 /** 350 * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) 351 * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo 352 * objects. 353 * @param numEntries The size of the array. 354 */ 355 void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries); 356 357 358 /** 359 * hdfsGetHosts - Get hostnames where a particular block (determined by 360 * pos & blocksize) of a file is stored. The last element in the array 361 * is NULL. Due to replication, a single block could be present on 362 * multiple hosts. 363 * @param fs The configured filesystem handle. 364 * @param path The path of the file. 365 * @param start The start of the block. 366 * @param length The length of the block. 367 * @return Returns a dynamically-allocated 2-d array of blocks-hosts; 368 * NULL on error. 369 */ 370 char*** hdfsGetHosts(hdfsFS fs, const char* path, 371 tOffset start, tOffset length); 372 373 374 /** 375 * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts 376 * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo 377 * objects. 378 * @param numEntries The size of the array. 379 */ 380 void hdfsFreeHosts(char ***blockHosts); 381 382 383 /** 384 * hdfsGetDefaultBlockSize - Get the optimum blocksize. 385 * @param fs The configured filesystem handle. 386 * @return Returns the blocksize; -1 on error. 387 */ 388 tOffset hdfsGetDefaultBlockSize(hdfsFS fs); 389 390 391 /** 392 * hdfsGetCapacity - Return the raw capacity of the filesystem. 393 * @param fs The configured filesystem handle. 394 * @return Returns the raw-capacity; -1 on error. 395 */ 396 tOffset hdfsGetCapacity(hdfsFS fs); 397 398 399 /** 400 * hdfsGetUsed - Return the total raw size of all files in the filesystem. 401 * @param fs The configured filesystem handle. 402 * @return Returns the total-size; -1 on error. 403 */ 404 tOffset hdfsGetUsed(hdfsFS fs); 405 406 #ifdef __cplusplus 407 } 408 #endif 409 410 #endif /*LIBHDFS_HDFS_H*/
