1. HDInsight
Programming

2. Port
• HDFS: http://localhost:50070/
• Oozie: http://localhost:11000/oozie/v1/
admin/status
• Templeton: http://localhost:50111/
templeton/v1/status
• ODBC: use port 10000 in DSN
configuration or connection string.

3. HDFS WebClient
Nuget Microsoft.NET API for Hadoop WebClient

4. WebHDFS

5. List Directory
var client = new WebHDFSClient(new Uri("http://localhost:50070"),"hadoop");!
client.GetDirectoryStatus("/").ContinueWith(dl => dl.Result.Directories.ToList().ForEach(d => Console.WriteLine("/" +
d.PathSuffix)));

6. Create Directory
var client = new WebHDFSClient(new Uri("http://localhost:50070"), "hadoop");!
var created = await client.CreateDirectory("/TEST");!
Console.WriteLine("True or False, we created the directory " + created.ToString());!
var deleted = await client.DeleteDirectory("/TEST");!
Console.WriteLine("True or False, we deleted the directory " + deleted.ToString());

7. Task Chaining
client.CreateDirectory("/TEST")!
.ContinueWith(x => client.CreateFile(@"c:tmpTitles.txt", "/user/hadoop/titles.txt")!
.ContinueWith(t => Console.WriteLine("new file located at " + t.Result))!
.ContinueWith(t => client.OpenFile("/user/hadoop/titles.txt")!
.ContinueWith(!
resp => resp.Result.Content.ReadAsStringAsync()!
.ContinueWith(bigString => Console.WriteLine("new file is " +
bigString.Result.Length + " bytes long"))!
.ContinueWith(!
t2 => client.DeleteDirectory("/user/hadoop/titles.txt")!
.ContinueWith(b => Console.WriteLine("Successfully deleted file."))!
)!
)!
)!
);

8. WebHCat
• Management of HCatalog metadata.
• Hive job submission.
• Pig job submission.
• Map/Reduce job submission.
• Streaming Map/Reduce job submission.

9. CreateHive
using System.Net.Http;
string outputDir = "basichivejob";!
var client = new WebHCatHttpClient(new Uri("http://localhost:50111"), "administrator", "", "hadoop");!
var t1 = client.CreateHiveJob(@"select * from src;", null, null, outputDir, null);!
t1.Wait();!
var response = t1.Result;!
var output = response.Content.ReadAsAsync<JObject>();!
output.Wait();!
response.EnsureSuccessStatusCode();!
string id = output.Result.GetValue("id").ToString();!
client.WaitForJobToCompleteAsync(id).Wait();

10. Oozie
http://hadoopsdk.codeplex.com/wikipage?title=Oozie
%20Client&referringTitle=Home

11. .NET MapReduce

12. MRRunner

13. Mapper
public class SqrtMapper : MapperBase!
{!
public override void Map(string inputLine, MapperContext context)!
{!
int inputValue = int.Parse(inputLine);!
!
// Perform the work.!
double sqrt = Math.Sqrt((double)inputValue);!
!
// Write output data.!
context.EmitKeyValue(inputValue.ToString(), sqrt.ToString());!
}!
}

14. Hadoop Job
public class FirstJob : HadoopJob<Mapper,Combiner,Reducer>!
{!
public override HadoopJobConfiguration Configure(ExecutorContext context)!
{!
HadoopJobConfiguration config = new HadoopJobConfiguration();!
config.InputPath = "input/SqrtJob";!
config.OutputFolder = "output/SqrtJob";!
return config;!
}!
}!

15. var hadoop = Hadoop.Connect();
hadoop.MapReduceJob.ExecuteJob<JobType>(arguments);

16. MRRunner -dll MyMRProgram.dll {-class jobClass} {-- job-class options}

17. Linq to Hive

18. HiveRow
public class TitlesRow : HiveRow!
{!
public string MovieId { get; set; }!
public string Name { get; set; }!
public int Year { get; set; }!
public string Rating { get; set; }!
}!
!
public class AwardsRow : HiveRow!
{!
public string MovieId { get; set; }!
public string AwardId { get; set; }!
public int Year { get; set; }!
public string Won { get; set; }!
public string Type { get; set; }!
public string Category { get; set; }!
}!
!
public class ActorsRow : HiveRow!
{!
public string MovieId { get; set; }!
public string ActorId { get; set; }!
public int AwardsCount { get; set; }!
public string Name { get; set; }!

19. HiveConnection
public class MyHiveDatabase : HiveConnection!
{!
public MyHiveDatabase(Uri webHcatUri, string username, string password, string azureStorageAccount, string azureStorageKey) : base(webHcatUri,
username, password, azureStorageAccount, azureStorageKey) { }!
!
public HiveTable<AwardsRow> Awards!
{!
get!
{!
return this.GetTable<AwardsRow>("Awards");!
}!
}!
!
public HiveTable<TitlesRow> Titles!
{!
get!
{!
return this.GetTable<TitlesRow>("Titles");!
}!
}!
!
public HiveTable<ActorsRow> Actors!
{!
get!
{!
return this.GetTable<ActorsRow>("Actors");!
}!
}!
}

20. Simple Linq
var db = new MyHiveDatabase(!
webHCatUri: new Uri("http://localhost:50111"),!
userName: "hadoop", password: null,!
storageAccount: “ASV storage account name”, storageKey: “ASV storage account key”);!
!
var q = from x in!
(from a in db.Actors!
select new { a.ActorId, foo = a.AwardsCount })!
group x by x.ActorId into g!
select new { ActorId = g.Key, bar = g.Average(z => z.foo) };!
!
q.ExecuteQuery().Wait();!
var results1 = q.ToList();!
!!
var projectionQuery = from aw in db.Awards!
join t in db.Titles!
on aw.MovieId equals t.MovieId!
where t.Year == 1994 && aw.Won == "True"!
select new { MovieId = t.MovieId, Name = t.Name, Type = aw.Type, Category = aw.Category,
Year = t.Year };!
!!
var newTable = projectionQuery.CreateTable("AwardsIn1994");

21. Excel ODBC

22. http://www.microsoft.com/en-us/download/details.aspx?
id=40886

34. Resource
• http://hadoopsdk.codeplex.com/
• https://github.com/WindowsAzure-Samples/
HDInsight-Labs-Preview
• http://wag.codeplex.com/

35. Mahout

36. Machine Learning is programming
computers to optimize a
performance criterion using
example data or past experience

38. Classification

39. Clustering

40. Recommenders

41. Collaborative Filtering -
User Based

42. Collaborative Filtering -
Item Based

43. Data
http://labrosa.ee.columbia.edu/millionsong/tasteprofile
http://www.grouplens.org/node/12

44. Mahout Command
c:appsdistmahout-0.7bin>hadoop jar c:Appsdistmahout-0.7mahout-core-0.7-job.jar
org.apache.mahout.cf.taste.hadoop.item.RecommenderJob -s SIMILARITY_COOCCURRENCE --input=input/mInput.txt --
output=output --usersFile=input/users.txt!