ASP.NET實現Hadoop增刪改查的示例代碼

<?xml version="1.0" encoding="utf-8"?>
<packages>
 <package id="Microsoft.AspNet.WebApi.Client" version="4.0.20505.0" targetFramework="net46" />
 <package id="Microsoft.Data.Edm" version="5.2.0" targetFramework="net46" />
 <package id="Microsoft.Data.OData" version="5.2.0" targetFramework="net46" />
 <package id="Microsoft.Hadoop.WebClient" version="0.12.5126.42915" targetFramework="net46" />
 <package id="Microsoft.Net.Http" version="2.0.20505.0" targetFramework="net46" />
 <package id="Microsoft.WindowsAzure.ConfigurationManager" version="1.8.0.0" targetFramework="net46" />
 <package id="Newtonsoft.Json" version="4.5.11" targetFramework="net46" />
 <package id="System.Spatial" version="5.2.0" targetFramework="net46" />
 <package id="WindowsAzure.Storage" version="2.0.4.1" targetFramework="net46" />
</packages>

using Microsoft.Hadoop.WebHDFS;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;

namespace Physical
{
 public sealed class HDFSAccess
 {
 private readonly WebHDFSClient webHDFSClient;

 public HDFSAccess(string uriString, string userName)
 {
 this.webHDFSClient = new WebHDFSClient(new Uri(uriString), userName);
 }

 public List<string> GetDirectories(string path)
 {
 var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

 return directoryStatus.Directories.Select(d => d.PathSuffix).ToList();
 }

 public List<string> GetFiles(string path)
 {
 var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

 return directoryStatus.Files.Select(d => d.PathSuffix).ToList();
 }

 public bool CreateDirectory(string path)
 {
 // 傳入路徑不包含根目錄時，預設會在根目錄「/」底下
 return this.webHDFSClient.CreateDirectory(path).Result;
 }

 public bool DeleteDirectory(string path)
 {
 // 傳入路徑不包含根目錄時，預設會在根目錄「/」底下
 return this.webHDFSClient.DeleteDirectory(path).Result;
 }

 public string CreateFile(string localFile, string remotePath)
 {
 // 傳入遠端路徑不包含根目錄時，預設會在根目錄「/」底下
 return this.webHDFSClient.CreateFile(localFile, remotePath).Result;
 }

 public bool DeleteFile(string path)
 {
 // 傳入路徑不包含根目錄時，預設會在根目錄「/」底下
 return this.webHDFSClient.DeleteDirectory(path).Result;
 }

 public HttpResponseMessage OpenFile(string path)
 {
 // 傳入路徑不包含根目錄時，預設會在根目錄「/」底下
 return this.webHDFSClient.OpenFile(path).Result;
 }
 }
}

using Physical;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace MyTest
{
 class Program
 {
 // HDFS cluster 客戶端進入端點設定于 主機上
 // 預設端點：http://[主機名稱]:50070
 // 預設帳號：Wu
 private static HDFSAccess access = new HDFSAccess(@"http://127.0.0.1:50070", "Wu");

 static void Main(string[] args)
 {
 GetDirectoriesTest();
 Console.WriteLine("----------------------------------------------");
 GetFilesTest();
 Console.WriteLine("----------------------------------------------");
 DirectoryTest();
 Console.WriteLine("----------------------------------------------");
 FileTest();
 Console.WriteLine("----------------------------------------------");
 OpenFileTest();
 Console.ReadKey();
 } 

 public void TestCleanup()
 {
 //取得根目錄資料夾
 var directories = access.GetDirectories(@"/");

 // 移除預設目錄：tmp、user 外的目錄
 foreach (var directory in directories)
 {
 if ("tmp".Equals(directory) || "user".Equals(directory))
 {
 continue;
 }
 else
 {
 access.DeleteDirectory(directory);
 }
 }

 // 取得根目錄檔案
 var files = access.GetFiles(@"/");

 // 移除所有檔案
 foreach (var file in files)
 {
 access.DeleteFile(file);
 }

 // 移除 OpenFile 轉存檔案
 File.Delete(Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg"));
 }

 //_傳入根目錄_預期回傳預設目錄
 public static void GetDirectoriesTest()
 {
 // 預設根目錄下有兩個目錄：tmp、user
 var expected = new List<string>() { "tmp", "user", };

 var actual = access.GetDirectories(@"/");

 foreach (var item in actual)
 {
 Console.WriteLine(item);
 }

 }

 //_傳入根目錄_預期回傳空集合
 public static void GetFilesTest()
 {
 // 預設根目錄下沒有檔案
 var expected = new List<string>();

 var actual = access.GetFiles(@"/");
 foreach (var item in actual)
 {
 Console.WriteLine(item);
 }

 }

 //_建立zzz目錄_預期成功_預期根目錄下有zzz目錄_刪除zzz目錄_預期成功_預期根目錄下無zzz目錄
 public static void DirectoryTest()
 {
 var directoryName = "zzz";

 // 建立zzz目錄
 var boolCreateDirectory = access.CreateDirectory(directoryName);

 Console.WriteLine("建立zzz目錄_預期成功:"+boolCreateDirectory);

 // 建立zzz目錄_預期成功_預期根目錄下有zzz目錄
 // 預設根目錄下有三個目錄：tmp、user、zzz
 var expectedCreateDirectory = new List<string>() { "tmp", "user", directoryName, };

 var actualCreateDirectory = access.GetDirectories(@"/");


 foreach (var item in actualCreateDirectory)
 {
 Console.WriteLine(item);
 }

 Console.WriteLine("********************************************");
 // 刪除zzz目錄
 var boolDeleteDirectory = access.DeleteDirectory(directoryName);

 Console.WriteLine("刪除zzz目錄_預期成功:" + boolDeleteDirectory);


 // 刪除zzz目錄_預期成功_預期根目錄下無zzz目錄
 // 預設根目錄下有兩個目錄：tmp、user
 var expectedDeleteDirectory = new List<string>() { "tmp", "user", };

 var actualDeleteDirectory = access.GetDirectories(@"/");

 foreach (var item in actualDeleteDirectory)
 {
 Console.WriteLine(item);
 }
 }

 //_建立Test檔案_預期根目錄下有Test檔案_刪除Test檔案_預期成功_預期根目錄下無Test檔案
 public static void FileTest()
 {
 var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");

 var remotePath = "Test.jpg";

 // 建立Test檔案
 var boolCreateFile = access.CreateFile(localFile, remotePath);

 // 建立Test檔案_預期根目錄下有Test檔案
 var expectedCreateFile = new List<string>() { remotePath, };

 var actualCreateFile = access.GetFiles(@"/");

 foreach (var item in actualCreateFile)
 {
 Console.WriteLine(item);
 }
 Console.WriteLine("********************************************");
 // 刪除Test檔案
 var boolDeleteFile = access.DeleteDirectory(remotePath);

 Console.WriteLine("刪除Test檔案_預期成功:"+boolDeleteFile);

 // 刪除Test檔案_預期成功_預期根目錄下無Test檔案
 var expectedDeleteFile = new List<string>();

 var actualDeleteFile = access.GetFiles(@"/");

 foreach (var item in actualDeleteFile)
 {
 Console.WriteLine(item);
 }
 }

 //_建立Test檔案_預期根目錄下有Test檔案_取得Test檔案_預期成功_預期回傳Test檔案Stream并轉存成功
 public static void OpenFileTest()
 {
 var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");
 var remotePath = "Test.jpg";
 var saveFile = Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg");

 Console.WriteLine("saveFile:" + saveFile);
 Console.WriteLine("********************************************");
 // 建立Test檔案
 var boolCreateFile = access.CreateFile(localFile, remotePath);

 // 建立Test檔案_預期根目錄下有Test檔案
 var expectedCreateFile = new List<string>() { remotePath, };

 var actualCreateFile = access.GetFiles(@"/");

 foreach (var item in actualCreateFile)
 {
 Console.WriteLine(item);
 }


 Console.WriteLine("********************************************");
 // 取得Test檔案
 var response = access.OpenFile(remotePath);

 // 取得Test檔案_預期成功
 response.EnsureSuccessStatusCode();

 // 取得Test檔案_預期成功_預期回傳Test檔案Stream并轉存成功
 using (var fs = File.Create(saveFile))
 {
 response.Content.CopyToAsync(fs).Wait();
 }

 Console.WriteLine(File.Exists(saveFile));

 }
 }
}