Integrating cortana with wp8 app

Community Day 2014
Kolkata Geeks

Abhishek Sur
Microsoft MVP in ASP.NET/IIS
Twitter : @abhi2434
Facebook : abhi2434
Email : contact@abhisheksur.com
Presented by

<?xml version="1.0" encoding="utf-8"?>
<VoiceCommands xmlns="http://schemas.microsoft.com/voicecommands/1.1">
<CommandSet xml:lang="en-us" Name="englishCommands">
<CommandPrefix>MSDN</CommandPrefix>
<Example>How do I add Voice Commands to my application</Example>
<Command Name="FindText">
<Example>Find Install Voice Command Sets</Example>
<ListenFor>Search</ListenFor>
<ListenFor>Search for {dictatedSearchTerms}</ListenFor>
<ListenFor>Find</ListenFor>
<ListenFor>Find {dictatedSearchTerms}</ListenFor>
<Feedback>Search on MSDN</Feedback>
<Navigate Target="MainPage.xaml" />
</Command>
<Command Name="nlpCommand">
<ListenFor>{dictatedVoiceCommandText}</ListenFor>
<Feedback>Starting MSDN...</Feedback>
</Command>
<PhraseTopic Label="dictatedVoiceCommandText" Scenario="Dictation">
<Subject>MSDN</Subject>
</PhraseTopic>
<PhraseTopic Label="dictatedSearchTerms" Scenario="Search">
</PhraseTopic>
</CommandSet>
</VoiceCommands>

</Command>
</Command>
</PhraseTopic>
</PhraseTopic>
</CommandSet>
</VoiceCommands>

</Command>
</Command>
</PhraseTopic>
</PhraseTopic>
</CommandSet>
</VoiceCommands>
</Command>
/Command>

</Command>
</Command>
</PhraseTopic>
</PhraseTopic>
</CommandSet>
</VoiceCommands>

</Command>
</Command>
</PhraseTopic>
</PhraseTopic>
</CommandSet>
</VoiceCommands>
</PhraseTopic>
</PhraseTopic>

</Command>
</Command>
</PhraseTopic>
</PhraseTopic>
</CommandSet>
</VoiceCommands>

Windows Phone Silverlight Apps on Windows Phone 8.1
private async void
// SHOULD BE PERFORMED UNDER TRY/CATCH
Uri new ms-appx:///vcd.xml UriKind.Absolute
await
Windows Runtime Apps on Windows Phone 8.1
private async void
// SHOULD BE PERFORMED UNDER TRY/CATCH
Uri uriVoiceCommands = new Uri("ms-appx:///vcd.xml", UriKind.Absolute);
StorageFile file = await StorageFile.GetFileFromApplicationUriAsync(uriVoiceCommands);
await VoiceCommandManager.InstallCommandSetsFromStorageFileAsync(file);

// Windows Phone Silverlight App, in MainPage.xaml.cs
protected override void OnNavigatedTo(System.Windows.Navigation.NavigationEventArgs e)
{
base.OnNavigatedTo(e);
if (e.NavigationMode == System.Windows.Navigation.NavigationMode.New)
{
string recoText = null; // What did the user say? e.g. MSDN, "Find Windows Phone Voice Commands"
NavigationContext.QueryString.TryGetValue("reco", out recoText);
string voiceCommandName = null; // Which command was recognized in the VCD.XML file? e.g. "FindText"
NavigationContext.QueryString.TryGetValue("voiceCommandName", out voiceCommandName);
string searchTerms = null; // What did the user say, for named phrase topic or list "slots"? e.g. "Windows Phone Voice Commands"
NavigationContext.QueryString.TryGetValue("dictatedSearchTerms", out searchTerms);
switch (voiceCommandName) // What command launched the app?
{
case "FindText":
HandleFindText(searchTerms);
break;
case "nlpCommand":
HandleNlpCommand(recoText);
break;
}
}
}

{
{
{
case "FindText":
break;
case "nlpCommand":
break;
}
}
}

{
{
{
case "FindText":
break;
case "nlpCommand":
break;
}
}
}
// What did the user say? e.g. MSDN, "Find Windows Phone Voice Commands"
string recoText = null;
// Which command was recognized in the VCD.XML file? e.g. "FindText"
string voiceCommandName = null;
// What did the user say, for named phrase topic or list "slots"? e.g. "Windows
Phone Voice Commands"
string searchTerms = null;

{
{
{
case "FindText":
break;
case "nlpCommand":
break;
}
}
}
{
case "FindText":
break;
case "nlpCommand":
break;
}

// Windows Runtime App on Windows Phone 8.1, inside OnActivated override in App class
if (args.Kind == ActivationKind.VoiceCommand)
{
VoiceCommandActivatedEventArgs vcArgs = (VoiceCommandActivatedEventArgs)args;
string voiceCommandName = vcArgs.Result.RulePath.First(); // What command launched the app?
switch (voiceCommandName) // Navigate to right page for the voice command
{
case "FindText": // User said "find" or "search"
rootFrame.Navigate(typeof(MSDN.FindText), vcArgs.Result);
break;
case "nlpCommand": // User said something else
rootFrame.Navigate(typeof(MSDN.NlpCommand), vcArgs.Result);
break;
}
}

{
{
break;
break;
}
}
// Windows Runtime App on Windows Phone 8.1, inside OnActivated override
// in App class

{
{
break;
break;
}
}
// What command launched the app?
string voiceCommandName = vcArgs.Result.RulePath.First();
// Navigate to right page for the voice command
switch (voiceCommandName)

{
{
break;
break;
}
}
// Navigate to right page for the voice command
switch (voiceCommandName)
{
break;
break;
}

// Windows Runtime App on Windows Phone 8.1, inside OnNavigatedTo in FindText.xaml.cs
protected override void OnNavigatedTo(NavigationEventArgs e)
{
// Get recognition result from parameter passed in frame.Navigate call
SpeechRecognitionResult vcResult = e.Parameter as SpeechRecognitionResult;
if (vcResult!=null)
{
string recoText = vcResult.Text;
// Store the semantics dictionary for later use
IReadOnlyDictionary<string,IReadOnlyList<string>> semantics = vcResult.SemanticInterpretation.Properties;
string voiceCommandName = vcResult.RulePath.First();
if (voiceCommandName == "FindText")
{
// What did the user say, for named phrase topic or list "slots"? e.g. "Windows Phone Voice Commands"
if (semantics.ContainsKey("dictatedSearchTerms"))
{
HandleFindTextWithSearchTerms(semantics["dictatedSearchTerms"][0]);
}
else
{
HandleNoSearchTerms();
}
}
// Else handle other voice commands
} navigationHelper.OnNavigatedTo(e)
}

// Windows Runtime App on Windows Phone 8.1, inside OnNavigatedTo In FindText.xaml.cs
{
if (vcResult!=null)
{
{
{
}
else
{
}
}
}
// Windows Runtime App on Windows Phone 8.1, inside
// OnNavigatedTo in FindText.xaml.cs
{
if (vcResult!=null)
{
IReadOnlyDictionary<string,IReadOnlyList<string>> semantics =
vcResult.SemanticInterpretation.Properties;

// Windows Runtime App on Windows Phone 8.1, inside OnNavigatedTo In FindText.xaml.cs
{
if (vcResult!=null)
{
{
{
}
else
{
}
}
}
{
// What did the user say, for named phrase topic or list "slots"?
// e.g. "Windows Phone Voice Commands"
{
}
else
{
}
}

// Windows Runtime App on Windows Phone 8.1, inside OnNavigatedTo in NlpCommand.xaml.cs
{
// Check for null!
string commandMode = vcResult.SemanticInterpretation.Properties["commandMode"][0];
if (commandMode == "voice") // Did the user speak or type the command?
{
SpeakText(audioPlayer, String.Format("MSDN app heard you say {0}", vcResult.Text));
HandleNlpCommand(vcResult);
}
else if(commandMode=="text")
{
messageTextBox.Text = string.Format("Working on your request "{0}"", vcResult.Text);
}
}

// Windows Runtime App on Windows Phone 8.1, inside OnNavigatedTo In NlpCommand.xaml.cs
{
// Check for null!
{
}
{
}
}
// Windows Runtime App on Windows Phone 8.1, in OnNavigatedTo in NlpCommand.xaml.cs
{

// Windows Runtime App on Windows Phone 8.1, inside OnNavigatedTo In NlpCommand.xaml.cs
{
// Check for null!
{
}
{
}
}
{
SpeakText(audioPlayer, String.Format("MSDN voice heard you say {0}", vcResult.Text));
}
{
msgTextBox.Text= string.Format("Working on your request "{0}"", vcResult.Text);
}

private void HandleNlpCommand(string recoText, bool actSilently)
{
string action = null;
string navigateTo = null;
string searchFor = null;
recoText = recoText.ToLower();
if (recoText.Contains("go to ") || recoText.Contains("goto ") ||
recoText.Contains("find ") || recoText.Contains("search ") ||
recoText.Contains("show me "))
{
action = "navigate";
if (recoText.Contains("windows phone dev center"))
{
navigateTo = "http://dev.windowsphone.com";
}
}
else if (recoText.Contains("learn how to "))
{
action = "find";
searchFor = recoText.Substring(recoText.IndexOf("learn how to ") + 13);
}
else
{
action = "find";
searchFor = recoText;
}
switch (action)
{
case "find":
// ...

{
{
{
}
}
{
action = "find";
}
else
{
action = "find";
}
switch (action)
{
case "find":
// ...
{
{
}
}

{
{
{
{
}
}
{
action = "find";
}
else
{
action = "find";
}
switch (action)
{
case "find":
// ...
action = "find";
}

{
else
{
{
{
}
}
{
action = "find";
}
else
{
action = "find";
}
switch (action)
{
case "find":
// ...
action = "find";
}

// Windows Phone Silverlight App
// Synthesis
private async void SpeakText(string textToSpeak)
{
SpeechSynthesizer synthesizer = new SpeechSynthesizer();
await synthesizer.SpeakTextAsync(textToSpeak);
}
// Recognition
private async Task<SpeechRecognitionUIResult> RecognizeSpeech()
{
SpeechRecognizerUI recognizer = new SpeechRecognizerUI();
// One of three Grammar types available
recognizer.Recognizer.Grammars.AddGrammarFromPredefinedType(
"key1", SpeechPredefinedGrammar.WebSearch);
await recognizer.Recognizer.PreloadGrammarsAsync(); // Optional but recommended
// Put up UI and recognize user's utterance
SpeechRecognitionUIResult result = await recognizer.RecognizeWithUIAsync();
return result;
}
// Calling code uses result.RecognitionResult.Text or result.RecognitionResult.Semantics

// Synthesis
{
}
// Synthesis
{
// Recognition
{
}
return result;
}

// Recognition
{
// Synthesis
{
}
// Recognition
{
// Optional but recommended
await recognizer.Recognizer.PreloadGrammarsAsync();
return result;
}

// Synthesis
{
}
SpeechRecognitionUIResult result
// Recognition
{
= await recognizer.RecognizeWithUIAsync();
return result;
return result;
}

// Windows Phone Store App
// Synthesis

<MediaElement Name="audioPlayer" AutoPlay="True" .../>
// C# code behind
// Function to speak a text string
private async void SpeakText(MediaElement audioPlayer, string textToSpeak)
{
SpeechSynthesisStream ttsStream = await synthesizer.SynthesizeTextToStreamAsync(textToSpeak);
audioPlayer.SetSource(ttsStream, ""); // This starts the player because AutoPlay="True"
}

// Synthesis
// C# code behind
{
}
// Synthesis

// C# code behind
{
// Synthesis
// C# code behind
{
}
SpeechSynthesisStream ttsStream
= await synthesizer.SynthesizeTextToStreamAsync(textToSpeak);
audioPlayer.SetSource(ttsStream, "");
// This starts the player because AutoPlay="True"
}

// Recognition
private async Task<SpeechRecognitionResult> RecognizeSpeech()
{
SpeechRecognizer recognizer = new SpeechRecognizer();
// One of three Constraint types available
SpeechRecognitionTopicConstraint topicConstraint
= new SpeechRecognitionTopicConstraint(SpeechRecognitionScenario.WebSearch, "MSDN");
recognizer.Constraints.Add(topicConstraint);
await recognizer.CompileConstraintsAsync(); // Required
SpeechRecognitionResult result = await recognizer.RecognizeWithUIAsync();
return result;
}
// Calling code uses result.RecognitionResult.Text or
// result.RecognitionResult.SemanticInterpretation

// Recognition
{
= new SpeechRecognitionTopicConstraint(
SpeechRecognitionScenario.WebSearch, "MSDN");
return result;
}

SpeechRecognitionResult result
// Recognition
{
= await recognizer.RecognizeWithUIAsync();
return result;
}
return result;
}

Integrating cortana with wp8 app

Integrating cortana with wp8 app

Recommended

Recommended

More Related Content

Viewers also liked

Viewers also liked (20)

Similar to Integrating cortana with wp8 app

Similar to Integrating cortana with wp8 app (20)

More from Abhishek Sur

More from Abhishek Sur (20)

Recently uploaded

Recently uploaded (20)

Integrating cortana with wp8 app