diff --git a/PVATestFramework/PVATestFramework/Helpers/Constants.cs b/PVATestFramework/PVATestFramework/Helpers/Constants.cs index 94a886d3..66cec173 100644 --- a/PVATestFramework/PVATestFramework/Helpers/Constants.cs +++ b/PVATestFramework/PVATestFramework/Helpers/Constants.cs @@ -16,6 +16,11 @@ public static class RoleTypes public const int Bot = 0; } + public static class Content + { + public const string BasicCard = "application/vnd.microsoft.card.hero"; + } + public static class Constants { public const string DataverseTokenUri = "https://login.microsoftonline.com/{TenantId}/oauth2/v2.0/token"; diff --git a/PVATestFramework/PVATestFramework/Helpers/Runner.cs b/PVATestFramework/PVATestFramework/Helpers/Runner.cs index 11eb94d8..4a9b37fc 100644 --- a/PVATestFramework/PVATestFramework/Helpers/Runner.cs +++ b/PVATestFramework/PVATestFramework/Helpers/Runner.cs @@ -1,813 +1,1544 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using PVATestFramework.Console.Helpers; -using PVATestFramework.Console.Helpers.DirectLine; -using PVATestFramework.Console.Helpers.Extensions; -using PVATestFramework.Console.Helpers.FileHandler; -using PVATestFramework.Console.Models; -using PVATestFramework.Console.Models.Activities; -using PVATestFramework.Console.Models.DirectLine; -using CsvHelper; -using CsvHelper.Configuration; -using Newtonsoft.Json; -using Serilog; -using System.Diagnostics; -using System.Globalization; -using System.Text.RegularExpressions; -using Activity = Microsoft.Bot.Connector.DirectLine.Activity; -using LoggerExtensions = PVATestFramework.Console.Helpers.Extensions.LoggerExtensions; -using Newtonsoft.Json.Linq; - -namespace PVATestFramework.Console -{ - public class Runner - { - private readonly ILogger logger; - private readonly IFileHandler fileHandler; - private readonly DirectLineClientBase directLineClient; - - public Runner(ILogger logger, IFileHandler fileHandler) - { - this.logger = logger; - this.fileHandler = fileHandler; - } - public Runner(ILogger logger, DirectLineClientBase directLineClient, IFileHandler fileHandler) - { - this.logger = logger; - this.directLineClient = directLineClient; - this.fileHandler = fileHandler; - } - - /// - /// Run a chat transcript test from a directory or file - /// - /// - /// - /// - /// - /// boolean depending if the execution was successful or not - public async Task RunTranscriptTestAsync(DirectLineOptions options, string path, bool verbose = false, CancellationToken cancellationToken = default) - { - logger.Information("The test has started..."); - - try - { - var totalTests = 0; - var failedTests = 0; - var fileAttributes = fileHandler.GetFileAttributes(path); - var fileList = new List(); - - if (fileAttributes.HasFlag(FileAttributes.Directory)) - { - // The path is a directory - fileList = fileHandler.GetFilesFromDirectory(path, "*.json").ToList(); - } - else - { - // The path is just a file - fileList.Add(path); - } - - logger.Information($"Using Direct Line endpoint: {options.RegionalEndpoint}"); - - foreach (var file in fileList) - { - logger.Information($"Testing with file: {file}"); - - var activityList = await GetActivitiesFromTranscriptFile(file); - - if (activityList.list_of_conversations.Count > 0) - { - // The file has multiple conversations - foreach (var actlist in activityList.list_of_conversations) - { - totalTests++; - var result = await ExecuteTranscriptAsync(options, actlist, file, verbose, cancellationToken).ConfigureAwait(false); - if (!result) failedTests++; - } - } - else - { - // The file has a single conversation - totalTests++; - var result = await ExecuteTranscriptAsync(options, activityList, file, verbose, cancellationToken).ConfigureAwait(false); - if (!result) failedTests++; - } - } - - logger.Information($"Test results: {totalTests - failedTests} passed, {totalTests} total"); - - return failedTests == 0; - } - catch (JsonReaderException ex) - { - logger.ForegroundColor($"An error occurred while running the test. Details: The json file used as an input is not valid.", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); - return false; - } - catch (Exception ex) - { - logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); - return false; - } - } - - /// - /// Run a scale test, sending N conversations to the bot - /// - /// - /// - /// - /// - /// boolean depending if the execution was successful or not - public async Task RunScaleTestAsync(DirectLineOptions options, int totalAttempts, string path, bool verbose) - { - logger.Information($"Running a Scale test (sending chat transcript {totalAttempts} times)."); - - try - { - var succeededAttempts = 0; - for (int i = 0; i < totalAttempts; i++) - { - if (verbose) - { - logger.Information($"Attempt {i+1}..."); - } - try - { - var result = await RunTranscriptTestAsync(options, path, verbose); - if (result) - { - succeededAttempts++; - } - } - catch - { - logger.ForegroundColor($"The chat transcript failed.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - } - } - - if (succeededAttempts == totalAttempts) - { - logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); - logger.ForegroundColor($"Scale test ended succesfully.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); - return true; - } - else - { - logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts from {totalAttempts}.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - logger.ForegroundColor($"Scale test ended with errors.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - return false; - } - } - catch (Exception ex) - { - logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); - return false; - } - } - - /// - /// Convert a .chat file into a .json file - /// - /// - /// - /// boolean depending if the conversion was successful or not - public bool ConvertChatFileToJSON(string path, string outputFile) - { - try - { - logger.Information($"The conversion process has started..."); - - var line = string.Empty; - var activityListContainer = new List>(); - var activityList = new List(); - string filepath = fileHandler.GetFullPath(outputFile); - - object? channelData = null; - - if (!filepath.EndsWith(".json")) - { - throw new ArgumentException("The output file should have a .json extension."); - } - if (!path.EndsWith(".chat")) - { - throw new ArgumentException("The input file should have a .chat extension."); - } - - using (StreamReader sr = new StreamReader(path)) - { - fileHandler.DeleteFile(filepath); - - while (null != (line = sr.ReadLine())) - { - var activity = new Models.Activities.Activity(); - if (string.IsNullOrWhiteSpace(line.Trim())) - { - continue; - } - else if (line.Trim().Equals("", StringComparison.InvariantCultureIgnoreCase)) - { - activityListContainer.Add(activityList); - activityList = new List(); - //reset channeldata for each conversation - channelData = null; - continue; - } - else if (line.StartsWith("user:")) - { - var userReg = new Regex(Regex.Escape("user:")); - var userText = userReg.Replace(line, string.Empty, 1).Trim(); - if (string.IsNullOrEmpty(userText)) - { - throw new ArgumentException("The user message is null or empty."); - } - else - { - activity = new Models.Activities.Activity - { - Type = Helpers.ActivityTypes.Message, - Text = userText, - From = new From(string.Empty, 1), - Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), - ChannelData = channelData - }; - } - } - else if (line.StartsWith("userEvent:")) - { - var userEventRegex = new Regex(Regex.Escape("userEvent:")); - var userEventText = userEventRegex.Replace(line, string.Empty, 1).Trim(); - - var userEventInfo = JObject.Parse(userEventText); - - if (string.IsNullOrEmpty(userEventText) || userEventInfo == null) - { - throw new ArgumentException("The userEvent is invalid"); - } - else - { - activity = new Models.Activities.Activity - { - Type = ActivityTypes.Event, - From = new From("user", 1), - Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), - ChannelData = channelData, - Name = (string)userEventInfo["Name"], - Value = (string)userEventInfo["Value"] - }; - } - } - else if (line.StartsWith("userMessage:")) - { - var userMessageRegex = new Regex(Regex.Escape("userMessage:")); - var userMessageText = userMessageRegex.Replace(line, string.Empty, 1).Trim(); - - var userMessageInfo = JObject.Parse(userMessageText); - - if (string.IsNullOrEmpty(userMessageText) || userMessageInfo == null) - { - throw new ArgumentException("The userMessage is invalid"); - } - else - { - activity = new Models.Activities.Activity - { - Type = ActivityTypes.Message, - From = new From("user", 1), - Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), - ChannelData = channelData, - Text = (string)userMessageInfo["Text"], - Value = (string)userMessageInfo["Value"] - }; - } - } - else if (line.StartsWith("bot:")) - { - var botReg = new Regex(Regex.Escape("bot:")); - var botText = botReg.Replace(line, string.Empty, 1).Trim(); - if (string.IsNullOrEmpty(botText)) - { - throw new ArgumentException("The bot message is null or empty."); - } - else - { - activity = new Models.Activities.Activity - { - Type = Helpers.ActivityTypes.Message, - Text = botText, - From = new From(string.Empty, 0), - Timestamp = ToUnixTimeSeconds(DateTime.UtcNow) - }; - } - } - else if (line.StartsWith("botEvent:")) - { - var botEventRegex = new Regex(Regex.Escape("botEvent:")); - var botEventText = botEventRegex.Replace(line, string.Empty, 1).Trim(); - - var botEventInfo = JObject.Parse(botEventText); - - if (string.IsNullOrEmpty(botEventText) || botEventInfo == null) - { - throw new ArgumentException("The botEvent is invalid"); - } - else - { - activity = new Models.Activities.Activity - { - Type = ActivityTypes.Event, - From = new From("bot", 0), - Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), - ChannelData = channelData, - Name = (string)botEventInfo["Name"], - Value = (string)botEventInfo["Value"] - }; - } - } - else if (line.StartsWith("suggested:")) - { - var suggestionRegex = new Regex(Regex.Escape("suggested:")); - var suggestionText = suggestionRegex.Replace(line, string.Empty, 1).Trim(); - - if (string.IsNullOrEmpty(suggestionText)) - { - throw new ArgumentException("The suggested message is null or empty."); - } - - var suggestionList = suggestionText.Split('|', StringSplitOptions.RemoveEmptyEntries).ToList(); - var intentCandidates = new List(); - - foreach (var suggestion in suggestionList) - { - intentCandidates.Add(new IntentCandidate - { - IntentScore = new IntentScore() - { - Title = suggestion - } - }); - } - activity = new Models.Activities.Activity - { - ValueType = "IntentCandidates", - Type = Helpers.ActivityTypes.Trace, - From = new From(string.Empty, 0), - Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), - Value = new Value() - { - IntentCandidates = intentCandidates - } - }; - } - else if (line.StartsWith("channelData:")) - { - var channelDataRegex = new Regex("channelData:"); - var channelDataText = channelDataRegex.Replace(line, string.Empty, 1).Trim(); - - if (string.IsNullOrEmpty(channelDataText)) - { - channelData = null; - continue; - } - else - { - channelData = JObject.Parse(channelDataText); - - // if this is the first thing in the conversation (nothing in the activity list yet), add a startConversation event - if(activityList.Count == 0) - { - activity = new Models.Activities.Activity - { - Type = Helpers.ActivityTypes.Event, - From = new From(string.Empty, 1), - ChannelData = channelData, - Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), - Name = "StartConversation" - }; - } - else - { - continue; - } - } - } - else - { - throw new Exception("The input file format is not valid."); - } - - activityList.Add(activity); - } - activityListContainer.Add(activityList); - - var activities = new List(); - foreach (var list in activityListContainer) - { - activities.Add($"{{\"Activities\":{string.Join(',', JsonConvert.SerializeObject(list))}}}"); - } - fileHandler.WriteToFile(filepath, $"{{\"list_of_conversations\":[{string.Join(",", activities)}]}}"); - } - - logger.Information($"The conversion process ended."); - return true; - } - catch (Exception ex) - { - logger.ForegroundColor($"An error occurred while converting the .chat transcript. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); - return false; - } - } - - /// - /// Execute the conversation on the transcript - /// - /// - /// - /// - /// - /// - /// boolean depending if the conversion was successful or not - /// - private async Task ExecuteTranscriptAsync(DirectLineOptions options, ActivityList activityList, string path, bool verbose = false, CancellationToken cancellationToken = default) - { - var logRecords = new List(); - var activities = new List(); - var timer = new Stopwatch(); - var testFailed = false; - var userUtterance = string.Empty; - - timer.Start(); - path = fileHandler.GetFullPath(path); - - try - { - using (directLineClient) - { - foreach (var activity in activityList.Activities) - { - switch (activity.From.Role) - { - case RoleTypes.User: - var sendActivity = new Activity - { - Type = activity.Type, - Text = activity.Text, - ChannelData = activity.ChannelData, - Name= activity.Name, - Value = activity.Value - }; - - userUtterance = sendActivity.Text; - if (verbose) - { - logger.Information($"User sends: {sendActivity.Text}"); - } - - await directLineClient.SendActivityAsync(sendActivity, cancellationToken).ConfigureAwait(false); - break; - - case RoleTypes.Bot: - if (IgnoreActivity(activity)) - { - break; - } - - var receivedActivity = new Activity(); - var receivedOptions = new List(); - var expectedOptions = new List(); - - if (activities.Count == 0) - { - activities = await directLineClient.ReceiveActivitiesAsync(cancellationToken).ConfigureAwait(false); - - // Get the first activity from the bot response - receivedActivity = activities.FirstOrDefault(); - activities.Remove(receivedActivity); - - if (verbose) - { - logger.Information($"Bot sends: {receivedActivity.Text}"); - } - if (receivedActivity.SuggestedActions != null) - { - // Get the suggested topics from the activity if any - receivedOptions = receivedActivity.SuggestedActions?.Actions?.Where(o => !o.Title.Equals(BotDefaultMessages.NoneOfThese, StringComparison.InvariantCultureIgnoreCase)).Select(a => a.Title).ToList(); - - if (verbose) - { - logger.Information($"\t{string.Join(" | ", receivedOptions)}"); - } - } - } - else - { - // Get the first activity from the list - receivedActivity = activities.FirstOrDefault(); - activities.Remove(receivedActivity); - - if (verbose) - { - logger.Information($"Bot sends: {receivedActivity.Text}"); - } - } - - var csvRecord = new LogCSV() - { - BotId = receivedActivity.From.Id, - ConversationId = receivedActivity.Conversation.Id, - SessionDate = DateTime.Now.ToString(), - UserUtterance = userUtterance, - ExpectedResponse = activity.Text, - ReceivedResponse = receivedActivity.Text, - TestFile = path - }; - - if (receivedActivity.Text != null && receivedActivity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase)) - { - // activity.Value redefined as type object?, so cast it back to type Value for this particular situation - expectedOptions = ((Value)(activity.Value))?.IntentCandidates != null ? ((Value)(activity.Value))?.IntentCandidates?.Select(o => o.IntentScore.Title).ToList() : new List() { "No suggested topics found" }; - - for (int i = 0; i < receivedOptions.Count; i++) - { - if (i == 0) csvRecord.DYM_Option1 = receivedOptions[i]; - if (i == 1) csvRecord.DYM_Option2 = receivedOptions[i]; - if (i == 2) csvRecord.DYM_Option3 = receivedOptions[i]; - } - - if (expectedOptions.Count != receivedOptions.Count) - { - testFailed = true; - } - else - { - foreach (var suggestion in receivedOptions) - { - if (!expectedOptions.Any(option => option.Equals(suggestion, StringComparison.InvariantCultureIgnoreCase))) - { - testFailed = true; - break; - } - } - } - - if (testFailed) - { - logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - logger.ForegroundColor($"Expected:\t{string.Join(" | ", expectedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - logger.ForegroundColor($"Received:\t{string.Join(" | ", receivedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - } - } - else - { - if (!AssertActivity(activity, receivedActivity)) - { - logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - - if (!string.IsNullOrEmpty(activity.Text)) - { - logger.ForegroundColor($"Expected: {activity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - logger.ForegroundColor($"Received: {receivedActivity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - logger.ForegroundColor($"Line number: {activity.LineNumber}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - } - - if (activity.Attachments != null && activity.Attachments.Count > 0) - { - var expectedAttachments = activity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); - var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); - - logger.ForegroundColor($"Expected: {string.Join(Environment.NewLine, expectedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - logger.ForegroundColor($"Received: {string.Join(Environment.NewLine, receivedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); - } - - testFailed = true; - } - } - - userUtterance = string.Empty; - csvRecord.Result = testFailed ? "Failed" : "Passed"; - logRecords.Add(csvRecord); - break; - - default: - throw new InvalidOperationException($"Invalid script role {activity.From.Role}."); - } - - if (testFailed) break; - } - } - - timer.Stop(); - - if (!testFailed) - { - logger.ForegroundColor($"Test script passed", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); - } - - logger.Information($"Time: {timer.Elapsed.TotalSeconds.ToString("0.00")} seconds"); - - return !testFailed; - } - catch (Exception ex) - { - logger.ForegroundColor($"An error occurred while validating the chat transcript file. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); - return false; - } - finally - { - WriteCSVLog(logRecords); - } - } - - /// - /// Receives an activityList from a transcript file - /// - /// - /// an activityList - private async Task GetActivitiesFromTranscriptFile(string fileName) - { - using var reader = new StreamReader(fileName); - var transcript = await reader.ReadToEndAsync().ConfigureAwait(false); - var activityList = JsonConvert.DeserializeObject(transcript); - - if (activityList.Activities.Count == 0 && activityList.list_of_conversations.Count == 0) - { - throw new JsonReaderException(); - } - else if (activityList.list_of_conversations.Count > 0) - { - foreach (var activities in activityList.list_of_conversations) - { - AddTextLineNumber(activities, fileName); - } - } - else - { - AddTextLineNumber(activityList, fileName); - } - - return activityList; - } - - /// - /// Gets the line number for the activity text - /// - /// - /// - private void AddTextLineNumber(ActivityList activityList, string fileName) - { - var lineNumber = 0; - var lines = File.ReadLines(fileName).ToList(); - - foreach (var activity in activityList.Activities.Where(a => a.IsMessageActivityWithText())) - { - // Added line number for Text messages - lineNumber = lines.FindIndex(lineNumber, line => line.Contains(activity.Text)) + 1; - activity.LineNumber = lineNumber; - } - } - - /// - /// Write CSV log - /// - /// - private void WriteCSVLog(List records) - { - var csvLogPath = Path.Combine(Environment.CurrentDirectory, "logFile.csv"); - var config = new CsvConfiguration(CultureInfo.InvariantCulture) - { - // Don't write the header again. - HasHeaderRecord = false, - // Use the system list separator - Delimiter = CultureInfo.CurrentCulture.TextInfo.ListSeparator - }; - - if (!File.Exists(csvLogPath)) - { - using (var streamWriter = new StreamWriter(csvLogPath)) - { - using (var csvWriter = new CsvWriter(streamWriter, config)) - { - csvWriter.WriteHeader(); - csvWriter.NextRecord(); - } - } - } - - using (var stream = File.Open(csvLogPath, FileMode.Append)) - { - using (var streamWriter = new StreamWriter(stream)) - { - using (var csvWriter = new CsvWriter(streamWriter, config)) - { - csvWriter.WriteRecords(records); - } - } - } - } - - /// - /// Check if adaptive cards structure are equal - /// - /// - /// - /// boolean - private bool AssertActivity(Models.Activities.Activity expectedActivity, Activity receivedActivity) - { - bool result = true; - if (!string.IsNullOrEmpty(expectedActivity.Text) && !string.IsNullOrEmpty(receivedActivity.Text)) - { - // Replace unwanted characters - receivedActivity.Text = receivedActivity.Text.Replace((char)0xA0, ' '); - - string pattern = ExtractRegex(expectedActivity.Text); - if (!string.IsNullOrEmpty(pattern)) - { - // If the line contains a regex pattern it will check if matches - return Regex.IsMatch(receivedActivity.Text, pattern); - } - // Allow for new lines in chat file, json and/or activity text by completely unescaping both strings - Issue 218 - else if (!expectedActivity.Text.CompletelyUnescape().Equals(receivedActivity.Text.CompletelyUnescape(), StringComparison.InvariantCultureIgnoreCase)) - { - // This is a simple text to compare - return false; - } - } - else if (expectedActivity.Attachments != null && expectedActivity.Attachments.Count == receivedActivity.Attachments.Count) - { - // This is an adaptive card, so the structure comparison will be executed - var expectedAttachments = expectedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); - var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); - var settings = new AdaptiveCardTranslatorSettings(); - - for (int i = 0; i < expectedActivity.Attachments.Count; i++) - { - var expectedCard = AdaptiveCard.GetCardWithoutValues(expectedAttachments[i].ToJObject(true), settings); - var receivedCard = AdaptiveCard.GetCardWithoutValues(receivedAttachments[i].ToJObject(true), settings); - if (!expectedCard.Equals(receivedCard, StringComparison.InvariantCultureIgnoreCase)) - { - return false; - } - } - } - else - { - return false; - } - - return result; - } - - /// - /// validate if the activity should be ignored - /// - /// - /// boolean - private bool IgnoreActivity(Models.Activities.Activity activity) - { - // Ignore trace activities unless it is an IntentCandidates type one. Also, ignore the DYM message as it is sent in the previous activity - return (activity.Type == Helpers.ActivityTypes.Trace - && !activity.ValueType.Equals("IntentCandidates", StringComparison.InvariantCultureIgnoreCase)) - || (activity.Type == Helpers.ActivityTypes.Message - && activity.Text != null - && activity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase)); - } - - /// - /// Extract the regex pattern - /// - /// - /// path - private string ExtractRegex(string input) - { - if (input == null) - { - return null; - } - string regexPattern = "<\\((.*?)\\)>"; // The regex pattern to search for - Regex regex = new Regex(regexPattern); - Match match = regex.Match(input); - if (match.Success) - { - return match.Groups[1].Value; // Extract the REGEX - } - else - { - return null; - } - } - - /// - /// Convert a datetime to Unix time format - /// - /// - /// path - private int ToUnixTimeSeconds(DateTime date) - { - DateTime point = new DateTime(1970, 1, 1); - TimeSpan time = date.Subtract(point); - - return (int)time.TotalSeconds; - } - } -} +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using PVATestFramework.Console.Helpers; +using PVATestFramework.Console.Helpers.DirectLine; +using PVATestFramework.Console.Helpers.Extensions; +using PVATestFramework.Console.Helpers.FileHandler; +using PVATestFramework.Console.Models; +using PVATestFramework.Console.Models.Activities; +using PVATestFramework.Console.Models.DirectLine; +using CsvHelper; +using CsvHelper.Configuration; +using Newtonsoft.Json; +using Serilog; +using System.Diagnostics; +using System.Globalization; +using System.Text.RegularExpressions; +using Activity = Microsoft.Bot.Connector.DirectLine.Activity; +using LoggerExtensions = PVATestFramework.Console.Helpers.Extensions.LoggerExtensions; + +namespace PVATestFramework.Console +{ + public class Runner + { + private readonly ILogger logger; + private readonly IFileHandler fileHandler; + private readonly DirectLineClientBase directLineClient; + + public Runner(ILogger logger, IFileHandler fileHandler) + { + this.logger = logger; + this.fileHandler = fileHandler; + } + public Runner(ILogger logger, DirectLineClientBase directLineClient, IFileHandler fileHandler) + { + this.logger = logger; + this.directLineClient = directLineClient; + this.fileHandler = fileHandler; + } + + /// + /// Run a chat transcript test from a directory or file + /// + /// + /// + /// + /// + /// boolean depending if the execution was successful or not + public async Task RunTranscriptTestAsync(DirectLineOptions options, string path, bool verbose = false, CancellationToken cancellationToken = default) + { + logger.Information("The test has started..."); + + try + { + var totalTests = 0; + var failedTests = 0; + var fileAttributes = fileHandler.GetFileAttributes(path); + var fileList = new List(); + + if (fileAttributes.HasFlag(FileAttributes.Directory)) + { + // The path is a directory + fileList = fileHandler.GetFilesFromDirectory(path, "*.json").ToList(); + } + else + { + // The path is just a file + fileList.Add(path); + } + + logger.Information($"Using Direct Line endpoint: {options.RegionalEndpoint}"); + + foreach (var file in fileList) + { + logger.Information($"Testing with file: {file}"); + + var activityList = await GetActivitiesFromTranscriptFile(file); + + if (activityList.list_of_conversations.Count > 0) + { + // The file has multiple conversations + foreach (var actlist in activityList.list_of_conversations) + { + totalTests++; + var result = await ExecuteTranscriptAsync(options, actlist, file, verbose, cancellationToken).ConfigureAwait(false); + if (!result) failedTests++; + } + } + else + { + // The file has a single conversation + totalTests++; + var result = await ExecuteTranscriptAsync(options, activityList, file, verbose, cancellationToken).ConfigureAwait(false); + if (!result) failedTests++; + } + } + + logger.Information($"Test results: {totalTests - failedTests} passed, {totalTests} total"); + + return failedTests == 0; + } + catch (JsonReaderException ex) + { + logger.ForegroundColor($"An error occurred while running the test. Details: The json file used as an input is not valid.", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + catch (Exception ex) + { + logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + } + + /// + /// Run a scale test, sending N conversations to the bot + /// + /// + /// + /// + /// + /// boolean depending if the execution was successful or not + public async Task RunScaleTestAsync(DirectLineOptions options, int totalAttempts, string path, bool verbose) + { + logger.Information($"Running a Scale test (sending chat transcript {totalAttempts} times)."); + + try + { + var succeededAttempts = 0; + for (int i = 0; i < totalAttempts; i++) + { + if (verbose) + { + logger.Information($"Attempt {i+1}..."); + } + try + { + var result = await RunTranscriptTestAsync(options, path, verbose); + if (result) + { + succeededAttempts++; + } + } + catch + { + logger.ForegroundColor($"The chat transcript failed.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + } + } + + if (succeededAttempts == totalAttempts) + { + logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); + logger.ForegroundColor($"Scale test ended succesfully.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); + return true; + } + else + { + logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts from {totalAttempts}.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Scale test ended with errors.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + return false; + } + } + catch (Exception ex) + { + logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + } + + /// + /// Convert a .chat file into a .json file + /// + /// + /// + /// boolean depending if the conversion was successful or not + public bool ConvertChatFileToJSON(string path, string outputFile) + { + try + { + logger.Information($"The conversion process has started..."); + + var line = string.Empty; + var activityListContainer = new List>(); + var activityList = new List(); + string filepath = fileHandler.GetFullPath(outputFile); + + if (!filepath.EndsWith(".json")) + { + throw new ArgumentException("The output file should have a .json extension."); + } + if (!path.EndsWith(".chat")) + { + throw new ArgumentException("The input file should have a .chat extension."); + } + + using (StreamReader sr = new StreamReader(path)) + { + fileHandler.DeleteFile(filepath); + + while (null != (line = sr.ReadLine())) + { + var activity = new Models.Activities.Activity(); + if (string.IsNullOrWhiteSpace(line.Trim())) + { + continue; + } + else if (line.Trim().Equals("", StringComparison.InvariantCultureIgnoreCase)) + { + activityListContainer.Add(activityList); + activityList = new List(); + continue; + } + else if (line.StartsWith("user:")) + { + var userReg = new Regex(Regex.Escape("user:")); + var userText = userReg.Replace(line, string.Empty, 1).Trim(); + if (string.IsNullOrEmpty(userText)) + { + throw new ArgumentException("The user message is null or empty."); + } + else + { + activity = new Models.Activities.Activity + { + Type = Helpers.ActivityTypes.Message, + Text = userText, + From = new From(string.Empty, 1), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow) + }; + } + } + else if (line.StartsWith("bot:")) + { + var botReg = new Regex(Regex.Escape("bot:")); + var botText = botReg.Replace(line, string.Empty, 1).Trim(); + if (string.IsNullOrEmpty(botText)) + { + throw new ArgumentException("The bot message is null or empty."); + } + else + { + activity = new Models.Activities.Activity + { + Type = Helpers.ActivityTypes.Message, + Text = botText, + From = new From(string.Empty, 0), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow) + }; + } + } + else if (line.StartsWith("suggested:")) + { + var suggestionRegex = new Regex(Regex.Escape("suggested:")); + var suggestionText = suggestionRegex.Replace(line, string.Empty, 1).Trim(); + + if (string.IsNullOrEmpty(suggestionText)) + { + throw new ArgumentException("The suggested message is null or empty."); + } + + var suggestionList = suggestionText.Split('|', StringSplitOptions.RemoveEmptyEntries).ToList(); + var intentCandidates = new List(); + + foreach (var suggestion in suggestionList) + { + intentCandidates.Add(new IntentCandidate + { + IntentScore = new IntentScore() + { + Title = suggestion + } + }); + } + activity = new Models.Activities.Activity + { + ValueType = "IntentCandidates", + Type = Helpers.ActivityTypes.Trace, + From = new From(string.Empty, 0), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), + Value = new Value() + { + IntentCandidates = intentCandidates + } + }; + } + else if (line.StartsWith("attachment:")) + { + var attachmentRegex = new Regex(Regex.Escape("attachment:")); + var attachmentTitle = attachmentRegex.Replace(line, string.Empty, 1).Trim(); + if (string.IsNullOrEmpty(attachmentTitle)){ + throw new ArgumentException("The attachment title is null or empty."); + } + else { + var lastActivityList = activityList.Last(); + lastActivityList.Attachments = [new Attachment(Helpers.Content.BasicCard, new BasicCardContent + { + Title = attachmentTitle, + Images = new List(), + Buttons = new List() + })]; + continue; + } + } + else + { + throw new Exception("The input file format is not valid."); + } + activityList.Add(activity); + } + activityListContainer.Add(activityList); + + var activities = new List(); + foreach (var list in activityListContainer) + { + activities.Add($"{{\"Activities\":{string.Join(',', JsonConvert.SerializeObject(list))}}}"); + } + fileHandler.WriteToFile(filepath, $"{{\"list_of_conversations\":[{string.Join(",", activities)}]}}"); + } + + logger.Information($"The conversion process ended."); + return true; + } + catch (Exception ex) + { + logger.ForegroundColor($"An error occurred while converting the .chat transcript. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + } + + /// + /// Execute the conversation on the transcript + /// + /// + /// + /// + /// + /// + /// boolean depending if the conversion was successful or not + /// + private async Task ExecuteTranscriptAsync(DirectLineOptions options, ActivityList activityList, string path, bool verbose = false, CancellationToken cancellationToken = default) + { + var logRecords = new List(); + var activities = new List(); + var timer = new Stopwatch(); + var testFailed = false; + var userUtterance = string.Empty; + + timer.Start(); + path = fileHandler.GetFullPath(path); + + try + { + using (directLineClient) + { + foreach (var activity in activityList.Activities) + { + switch (activity.From.Role) + { + case RoleTypes.User: + var sendActivity = new Activity + { + Type = activity.Type, + Text = activity.Text + }; + + userUtterance = sendActivity.Text; + if (verbose) + { + logger.Information($"User sends: {sendActivity.Text}"); + } + + await directLineClient.SendActivityAsync(sendActivity, cancellationToken).ConfigureAwait(false); + break; + + case RoleTypes.Bot: + if (IgnoreActivity(activity)) + { + break; + } + + var receivedActivity = new Activity(); + var receivedOptions = new List(); + var expectedOptions = new List(); + + if (activities.Count == 0) + { + activities = await directLineClient.ReceiveActivitiesAsync(cancellationToken).ConfigureAwait(false); + + // Get the first activity from the bot response + receivedActivity = activities.FirstOrDefault(); + activities.Remove(receivedActivity); + + if (verbose) + { + logger.Information($"Bot sends: {receivedActivity.Text}"); + } + if (receivedActivity.SuggestedActions != null) + { + // Get the suggested topics from the activity if any + receivedOptions = receivedActivity.SuggestedActions?.Actions?.Where(o => !o.Title.Equals(BotDefaultMessages.NoneOfThese, StringComparison.InvariantCultureIgnoreCase)).Select(a => a.Title).ToList(); + + if (verbose) + { + logger.Information($"\t{string.Join(" | ", receivedOptions)}"); + } + } + } + else + { + // Get the first activity from the list + receivedActivity = activities.FirstOrDefault(); + activities.Remove(receivedActivity); + + if (verbose) + { + logger.Information($"Bot sends: {receivedActivity.Text}"); + } + } + + var csvRecord = new LogCSV() + { + BotId = receivedActivity.From.Id, + ConversationId = receivedActivity.Conversation.Id, + SessionDate = DateTime.Now.ToString(), + UserUtterance = userUtterance, + ExpectedResponse = activity.Text, + ReceivedResponse = receivedActivity.Text, + TestFile = path + }; + + if (receivedActivity.Text != null && receivedActivity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase)) + { + expectedOptions = activity.Value?.IntentCandidates != null ? activity.Value?.IntentCandidates?.Select(o => o.IntentScore.Title).ToList() : new List() { "No suggested topics found" }; + + for (int i = 0; i < receivedOptions.Count; i++) + { + if (i == 0) csvRecord.DYM_Option1 = receivedOptions[i]; + if (i == 1) csvRecord.DYM_Option2 = receivedOptions[i]; + if (i == 2) csvRecord.DYM_Option3 = receivedOptions[i]; + } + + if (expectedOptions.Count != receivedOptions.Count) + { + testFailed = true; + } + else + { + foreach (var suggestion in receivedOptions) + { + if (!expectedOptions.Any(option => option.Equals(suggestion, StringComparison.InvariantCultureIgnoreCase))) + { + testFailed = true; + break; + } + } + } + + if (testFailed) + { + logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Expected:\t{string.Join(" | ", expectedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Received:\t{string.Join(" | ", receivedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + } + } + else + { + if (!AssertActivity(activity, receivedActivity)) + { + logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + + if (!string.IsNullOrEmpty(activity.Text)) + { + logger.ForegroundColor($"Expected: {activity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Received: {receivedActivity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Line number: {activity.LineNumber}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + } + + + testFailed = true; + } + if (activity.Attachments != null && activity.Attachments.Count > 0 && !AssertActivityAttachment(activity, receivedActivity)) + { + logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + var expectedAttachments = activity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); + var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); + logger.ForegroundColor($"Expected: {string.Join(Environment.NewLine, expectedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Received: {string.Join(Environment.NewLine, receivedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + testFailed = true; + } + } + userUtterance = string.Empty; + csvRecord.Result = testFailed ? "Failed" : "Passed"; + logRecords.Add(csvRecord); + break; + + default: + throw new InvalidOperationException($"Invalid script role {activity.From.Role}."); + } + + if (testFailed) break; + } + } + + timer.Stop(); + + if (!testFailed) + { + logger.ForegroundColor($"Test script passed", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); + } + + logger.Information($"Time: {timer.Elapsed.TotalSeconds.ToString("0.00")} seconds"); + + return !testFailed; + } + catch (Exception ex) + { + logger.ForegroundColor($"An error occurred while validating the chat transcript file. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + finally + { + WriteCSVLog(logRecords); + } + } + + /// + /// Receives an activityList from a transcript file + /// + /// + /// an activityList + private async Task GetActivitiesFromTranscriptFile(string fileName) + { + using var reader = new StreamReader(fileName); + var transcript = await reader.ReadToEndAsync().ConfigureAwait(false); + var activityList = JsonConvert.DeserializeObject(transcript); + + if (activityList.Activities.Count == 0 && activityList.list_of_conversations.Count == 0) + { + throw new JsonReaderException(); + } + else if (activityList.list_of_conversations.Count > 0) + { + foreach (var activities in activityList.list_of_conversations) + { + AddTextLineNumber(activities, fileName); + } + } + else + { + AddTextLineNumber(activityList, fileName); + } + + return activityList; + } + + /// + /// Gets the line number for the activity text + /// + /// + /// + private void AddTextLineNumber(ActivityList activityList, string fileName) + { + var lineNumber = 0; + var lines = File.ReadLines(fileName).ToList(); + + foreach (var activity in activityList.Activities.Where(a => a.IsMessageActivityWithText())) + { + // Added line number for Text messages + lineNumber = lines.FindIndex(lineNumber, line => line.Contains(activity.Text)) + 1; + activity.LineNumber = lineNumber; + } + } + + /// + /// Write CSV log + /// + /// + private void WriteCSVLog(List records) + { + var csvLogPath = Path.Combine(Environment.CurrentDirectory, "logFile.csv"); + var config = new CsvConfiguration(CultureInfo.InvariantCulture) + { + // Don't write the header again. + HasHeaderRecord = false, + // Use the system list separator + Delimiter = CultureInfo.CurrentCulture.TextInfo.ListSeparator + }; + + if (!File.Exists(csvLogPath)) + { + using (var streamWriter = new StreamWriter(csvLogPath)) + { + using (var csvWriter = new CsvWriter(streamWriter, config)) + { + csvWriter.WriteHeader(); + csvWriter.NextRecord(); + } + } + } + + using (var stream = File.Open(csvLogPath, FileMode.Append)) + { + using (var streamWriter = new StreamWriter(stream)) + { + using (var csvWriter = new CsvWriter(streamWriter, config)) + { + csvWriter.WriteRecords(records); + } + } + } + } + + /// + /// Check if adaptive cards structure are equal + /// + /// + /// + /// boolean + private bool AssertActivity(Models.Activities.Activity expectedActivity, Activity receivedActivity) + { + bool result = true; + if (!string.IsNullOrEmpty(expectedActivity.Text) && !string.IsNullOrEmpty(receivedActivity.Text)) + { + // Replace unwanted characters + receivedActivity.Text = receivedActivity.Text.Replace((char)0xA0, ' '); + + string pattern = ExtractRegex(expectedActivity.Text); + if (!string.IsNullOrEmpty(pattern)) + { + // If the line contains a regex pattern it will check if matches + return Regex.IsMatch(receivedActivity.Text, pattern); + } + // Allow for new lines in chat file, json and/or activity text by completely unescaping both strings - Issue 218 + else if (!expectedActivity.Text.CompletelyUnescape().Equals(receivedActivity.Text.CompletelyUnescape(), StringComparison.InvariantCultureIgnoreCase)) + { + // This is a simple text to compare + return false; + } + } + else + { + return false; + } + return result; + } + + private bool AssertActivityAttachment(Models.Activities.Activity expectedActivity, Activity receivedActivity) + { + bool result = true; + if (expectedActivity.Attachments != null && expectedActivity.Attachments.Count == receivedActivity.Attachments.Count) + { + // This is an adaptive card, so the structure comparison will be executed + var expectedAttachments = expectedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); + var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); + var settings = new AdaptiveCardTranslatorSettings(); + for (int i = 0; i < expectedActivity.Attachments.Count; i++) + { + var expectedCard = AdaptiveCard.GetCardWithoutValues(expectedAttachments[i].ToJObject(true), settings); + var receivedCard = AdaptiveCard.GetCardWithoutValues(receivedAttachments[i].ToJObject(true), settings); + logger.Information($"expected card: {expectedAttachments[i]}"); + logger.Information($"received card: {receivedAttachments[i]}"); + if (!expectedCard.Equals(receivedCard, StringComparison.InvariantCultureIgnoreCase)) + { + return false; + } + if (!expectedAttachments[i].Equals(receivedAttachments[i], StringComparison.InvariantCultureIgnoreCase) && expectedActivity.Attachments[i].ContentType == Helpers.Content.BasicCard) + { + return false; + } + } + } + else + { + return false; + } + return result; + } + + /// + /// validate if the activity should be ignored + /// + /// + /// boolean + private bool IgnoreActivity(Models.Activities.Activity activity) + { + // Ignore trace activities unless it is an IntentCandidates type one. Also, ignore the DYM message as it is sent in the previous activity + return (activity.Type == Helpers.ActivityTypes.Trace + && !activity.ValueType.Equals("IntentCandidates", StringComparison.InvariantCultureIgnoreCase)) + || (activity.Type == Helpers.ActivityTypes.Message + && activity.Text != null + && activity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase)); + } + + /// + /// Extract the regex pattern + /// + /// + /// path + private string ExtractRegex(string input) + { + if (input == null) + { + return null; + } + string regexPattern = "<\\((.*?)\\)>"; // The regex pattern to search for + Regex regex = new Regex(regexPattern); + Match match = regex.Match(input); + if (match.Success) + { + return match.Groups[1].Value; // Extract the REGEX + } + else + { + return null; + } + } + + /// + /// Convert a datetime to Unix time format + /// + /// + /// path + private int ToUnixTimeSeconds(DateTime date) + { + DateTime point = new DateTime(1970, 1, 1); + TimeSpan time = date.Subtract(point); + + return (int)time.TotalSeconds; + } + } +} +======= +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using PVATestFramework.Console.Helpers; +using PVATestFramework.Console.Helpers.DirectLine; +using PVATestFramework.Console.Helpers.Extensions; +using PVATestFramework.Console.Helpers.FileHandler; +using PVATestFramework.Console.Models; +using PVATestFramework.Console.Models.Activities; +using PVATestFramework.Console.Models.DirectLine; +using CsvHelper; +using CsvHelper.Configuration; +using Newtonsoft.Json; +using Serilog; +using System.Diagnostics; +using System.Globalization; +using System.Text.RegularExpressions; +using Activity = Microsoft.Bot.Connector.DirectLine.Activity; +using LoggerExtensions = PVATestFramework.Console.Helpers.Extensions.LoggerExtensions; +using Newtonsoft.Json.Linq; + +namespace PVATestFramework.Console +{ + public class Runner + { + private readonly ILogger logger; + private readonly IFileHandler fileHandler; + private readonly DirectLineClientBase directLineClient; + + public Runner(ILogger logger, IFileHandler fileHandler) + { + this.logger = logger; + this.fileHandler = fileHandler; + } + public Runner(ILogger logger, DirectLineClientBase directLineClient, IFileHandler fileHandler) + { + this.logger = logger; + this.directLineClient = directLineClient; + this.fileHandler = fileHandler; + } + + /// + /// Run a chat transcript test from a directory or file + /// + /// + /// + /// + /// + /// boolean depending if the execution was successful or not + public async Task RunTranscriptTestAsync(DirectLineOptions options, string path, bool verbose = false, CancellationToken cancellationToken = default) + { + logger.Information("The test has started..."); + + try + { + var totalTests = 0; + var failedTests = 0; + var fileAttributes = fileHandler.GetFileAttributes(path); + var fileList = new List(); + + if (fileAttributes.HasFlag(FileAttributes.Directory)) + { + // The path is a directory + fileList = fileHandler.GetFilesFromDirectory(path, "*.json").ToList(); + } + else + { + // The path is just a file + fileList.Add(path); + } + + logger.Information($"Using Direct Line endpoint: {options.RegionalEndpoint}"); + + foreach (var file in fileList) + { + logger.Information($"Testing with file: {file}"); + + var activityList = await GetActivitiesFromTranscriptFile(file); + + if (activityList.list_of_conversations.Count > 0) + { + // The file has multiple conversations + foreach (var actlist in activityList.list_of_conversations) + { + totalTests++; + var result = await ExecuteTranscriptAsync(options, actlist, file, verbose, cancellationToken).ConfigureAwait(false); + if (!result) failedTests++; + } + } + else + { + // The file has a single conversation + totalTests++; + var result = await ExecuteTranscriptAsync(options, activityList, file, verbose, cancellationToken).ConfigureAwait(false); + if (!result) failedTests++; + } + } + + logger.Information($"Test results: {totalTests - failedTests} passed, {totalTests} total"); + + return failedTests == 0; + } + catch (JsonReaderException ex) + { + logger.ForegroundColor($"An error occurred while running the test. Details: The json file used as an input is not valid.", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + catch (Exception ex) + { + logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + } + + /// + /// Run a scale test, sending N conversations to the bot + /// + /// + /// + /// + /// + /// boolean depending if the execution was successful or not + public async Task RunScaleTestAsync(DirectLineOptions options, int totalAttempts, string path, bool verbose) + { + logger.Information($"Running a Scale test (sending chat transcript {totalAttempts} times)."); + + try + { + var succeededAttempts = 0; + for (int i = 0; i < totalAttempts; i++) + { + if (verbose) + { + logger.Information($"Attempt {i+1}..."); + } + try + { + var result = await RunTranscriptTestAsync(options, path, verbose); + if (result) + { + succeededAttempts++; + } + } + catch + { + logger.ForegroundColor($"The chat transcript failed.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + } + } + + if (succeededAttempts == totalAttempts) + { + logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); + logger.ForegroundColor($"Scale test ended succesfully.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); + return true; + } + else + { + logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts from {totalAttempts}.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Scale test ended with errors.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + return false; + } + } + catch (Exception ex) + { + logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + } + + /// + /// Convert a .chat file into a .json file + /// + /// + /// + /// boolean depending if the conversion was successful or not + public bool ConvertChatFileToJSON(string path, string outputFile) + { + try + { + logger.Information($"The conversion process has started..."); + + var line = string.Empty; + var activityListContainer = new List>(); + var activityList = new List(); + string filepath = fileHandler.GetFullPath(outputFile); + + object? channelData = null; + + if (!filepath.EndsWith(".json")) + { + throw new ArgumentException("The output file should have a .json extension."); + } + if (!path.EndsWith(".chat")) + { + throw new ArgumentException("The input file should have a .chat extension."); + } + + using (StreamReader sr = new StreamReader(path)) + { + fileHandler.DeleteFile(filepath); + + while (null != (line = sr.ReadLine())) + { + var activity = new Models.Activities.Activity(); + if (string.IsNullOrWhiteSpace(line.Trim())) + { + continue; + } + else if (line.Trim().Equals("", StringComparison.InvariantCultureIgnoreCase)) + { + activityListContainer.Add(activityList); + activityList = new List(); + //reset channeldata for each conversation + channelData = null; + continue; + } + else if (line.StartsWith("user:")) + { + var userReg = new Regex(Regex.Escape("user:")); + var userText = userReg.Replace(line, string.Empty, 1).Trim(); + if (string.IsNullOrEmpty(userText)) + { + throw new ArgumentException("The user message is null or empty."); + } + else + { + activity = new Models.Activities.Activity + { + Type = Helpers.ActivityTypes.Message, + Text = userText, + From = new From(string.Empty, 1), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), + ChannelData = channelData + }; + } + } + else if (line.StartsWith("userEvent:")) + { + var userEventRegex = new Regex(Regex.Escape("userEvent:")); + var userEventText = userEventRegex.Replace(line, string.Empty, 1).Trim(); + + var userEventInfo = JObject.Parse(userEventText); + + if (string.IsNullOrEmpty(userEventText) || userEventInfo == null) + { + throw new ArgumentException("The userEvent is invalid"); + } + else + { + activity = new Models.Activities.Activity + { + Type = ActivityTypes.Event, + From = new From("user", 1), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), + ChannelData = channelData, + Name = (string)userEventInfo["Name"], + Value = (string)userEventInfo["Value"] + }; + } + } + else if (line.StartsWith("userMessage:")) + { + var userMessageRegex = new Regex(Regex.Escape("userMessage:")); + var userMessageText = userMessageRegex.Replace(line, string.Empty, 1).Trim(); + + var userMessageInfo = JObject.Parse(userMessageText); + + if (string.IsNullOrEmpty(userMessageText) || userMessageInfo == null) + { + throw new ArgumentException("The userMessage is invalid"); + } + else + { + activity = new Models.Activities.Activity + { + Type = ActivityTypes.Message, + From = new From("user", 1), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), + ChannelData = channelData, + Text = (string)userMessageInfo["Text"], + Value = (string)userMessageInfo["Value"] + }; + } + } + else if (line.StartsWith("bot:")) + { + var botReg = new Regex(Regex.Escape("bot:")); + var botText = botReg.Replace(line, string.Empty, 1).Trim(); + if (string.IsNullOrEmpty(botText)) + { + throw new ArgumentException("The bot message is null or empty."); + } + else + { + activity = new Models.Activities.Activity + { + Type = Helpers.ActivityTypes.Message, + Text = botText, + From = new From(string.Empty, 0), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow) + }; + } + } + else if (line.StartsWith("botEvent:")) + { + var botEventRegex = new Regex(Regex.Escape("botEvent:")); + var botEventText = botEventRegex.Replace(line, string.Empty, 1).Trim(); + + var botEventInfo = JObject.Parse(botEventText); + + if (string.IsNullOrEmpty(botEventText) || botEventInfo == null) + { + throw new ArgumentException("The botEvent is invalid"); + } + else + { + activity = new Models.Activities.Activity + { + Type = ActivityTypes.Event, + From = new From("bot", 0), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), + ChannelData = channelData, + Name = (string)botEventInfo["Name"], + Value = (string)botEventInfo["Value"] + }; + } + } + else if (line.StartsWith("suggested:")) + { + var suggestionRegex = new Regex(Regex.Escape("suggested:")); + var suggestionText = suggestionRegex.Replace(line, string.Empty, 1).Trim(); + + if (string.IsNullOrEmpty(suggestionText)) + { + throw new ArgumentException("The suggested message is null or empty."); + } + + var suggestionList = suggestionText.Split('|', StringSplitOptions.RemoveEmptyEntries).ToList(); + var intentCandidates = new List(); + + foreach (var suggestion in suggestionList) + { + intentCandidates.Add(new IntentCandidate + { + IntentScore = new IntentScore() + { + Title = suggestion + } + }); + } + activity = new Models.Activities.Activity + { + ValueType = "IntentCandidates", + Type = Helpers.ActivityTypes.Trace, + From = new From(string.Empty, 0), + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), + Value = new Value() + { + IntentCandidates = intentCandidates + } + }; + } + else if (line.StartsWith("channelData:")) + { + var channelDataRegex = new Regex("channelData:"); + var channelDataText = channelDataRegex.Replace(line, string.Empty, 1).Trim(); + + if (string.IsNullOrEmpty(channelDataText)) + { + channelData = null; + continue; + } + else + { + channelData = JObject.Parse(channelDataText); + + // if this is the first thing in the conversation (nothing in the activity list yet), add a startConversation event + if(activityList.Count == 0) + { + activity = new Models.Activities.Activity + { + Type = Helpers.ActivityTypes.Event, + From = new From(string.Empty, 1), + ChannelData = channelData, + Timestamp = ToUnixTimeSeconds(DateTime.UtcNow), + Name = "StartConversation" + }; + } + else + { + continue; + } + } + } + else + { + throw new Exception("The input file format is not valid."); + } + + activityList.Add(activity); + } + activityListContainer.Add(activityList); + + var activities = new List(); + foreach (var list in activityListContainer) + { + activities.Add($"{{\"Activities\":{string.Join(',', JsonConvert.SerializeObject(list))}}}"); + } + fileHandler.WriteToFile(filepath, $"{{\"list_of_conversations\":[{string.Join(",", activities)}]}}"); + } + + logger.Information($"The conversion process ended."); + return true; + } + catch (Exception ex) + { + logger.ForegroundColor($"An error occurred while converting the .chat transcript. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + } + + /// + /// Execute the conversation on the transcript + /// + /// + /// + /// + /// + /// + /// boolean depending if the conversion was successful or not + /// + private async Task ExecuteTranscriptAsync(DirectLineOptions options, ActivityList activityList, string path, bool verbose = false, CancellationToken cancellationToken = default) + { + var logRecords = new List(); + var activities = new List(); + var timer = new Stopwatch(); + var testFailed = false; + var userUtterance = string.Empty; + + timer.Start(); + path = fileHandler.GetFullPath(path); + + try + { + using (directLineClient) + { + foreach (var activity in activityList.Activities) + { + switch (activity.From.Role) + { + case RoleTypes.User: + var sendActivity = new Activity + { + Type = activity.Type, + Text = activity.Text, + ChannelData = activity.ChannelData, + Name= activity.Name, + Value = activity.Value + }; + + userUtterance = sendActivity.Text; + if (verbose) + { + logger.Information($"User sends: {sendActivity.Text}"); + } + + await directLineClient.SendActivityAsync(sendActivity, cancellationToken).ConfigureAwait(false); + break; + + case RoleTypes.Bot: + if (IgnoreActivity(activity)) + { + break; + } + + var receivedActivity = new Activity(); + var receivedOptions = new List(); + var expectedOptions = new List(); + + if (activities.Count == 0) + { + activities = await directLineClient.ReceiveActivitiesAsync(cancellationToken).ConfigureAwait(false); + + // Get the first activity from the bot response + receivedActivity = activities.FirstOrDefault(); + activities.Remove(receivedActivity); + + if (verbose) + { + logger.Information($"Bot sends: {receivedActivity.Text}"); + } + if (receivedActivity.SuggestedActions != null) + { + // Get the suggested topics from the activity if any + receivedOptions = receivedActivity.SuggestedActions?.Actions?.Where(o => !o.Title.Equals(BotDefaultMessages.NoneOfThese, StringComparison.InvariantCultureIgnoreCase)).Select(a => a.Title).ToList(); + + if (verbose) + { + logger.Information($"\t{string.Join(" | ", receivedOptions)}"); + } + } + } + else + { + // Get the first activity from the list + receivedActivity = activities.FirstOrDefault(); + activities.Remove(receivedActivity); + + if (verbose) + { + logger.Information($"Bot sends: {receivedActivity.Text}"); + } + } + + var csvRecord = new LogCSV() + { + BotId = receivedActivity.From.Id, + ConversationId = receivedActivity.Conversation.Id, + SessionDate = DateTime.Now.ToString(), + UserUtterance = userUtterance, + ExpectedResponse = activity.Text, + ReceivedResponse = receivedActivity.Text, + TestFile = path + }; + + if (receivedActivity.Text != null && receivedActivity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase)) + { + // activity.Value redefined as type object?, so cast it back to type Value for this particular situation + expectedOptions = ((Value)(activity.Value))?.IntentCandidates != null ? ((Value)(activity.Value))?.IntentCandidates?.Select(o => o.IntentScore.Title).ToList() : new List() { "No suggested topics found" }; + + for (int i = 0; i < receivedOptions.Count; i++) + { + if (i == 0) csvRecord.DYM_Option1 = receivedOptions[i]; + if (i == 1) csvRecord.DYM_Option2 = receivedOptions[i]; + if (i == 2) csvRecord.DYM_Option3 = receivedOptions[i]; + } + + if (expectedOptions.Count != receivedOptions.Count) + { + testFailed = true; + } + else + { + foreach (var suggestion in receivedOptions) + { + if (!expectedOptions.Any(option => option.Equals(suggestion, StringComparison.InvariantCultureIgnoreCase))) + { + testFailed = true; + break; + } + } + } + + if (testFailed) + { + logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Expected:\t{string.Join(" | ", expectedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Received:\t{string.Join(" | ", receivedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + } + } + else + { + if (!AssertActivity(activity, receivedActivity)) + { + logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + + if (!string.IsNullOrEmpty(activity.Text)) + { + logger.ForegroundColor($"Expected: {activity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Received: {receivedActivity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Line number: {activity.LineNumber}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + } + + if (activity.Attachments != null && activity.Attachments.Count > 0) + { + var expectedAttachments = activity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); + var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); + + logger.ForegroundColor($"Expected: {string.Join(Environment.NewLine, expectedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + logger.ForegroundColor($"Received: {string.Join(Environment.NewLine, receivedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow); + } + + testFailed = true; + } + } + + userUtterance = string.Empty; + csvRecord.Result = testFailed ? "Failed" : "Passed"; + logRecords.Add(csvRecord); + break; + + default: + throw new InvalidOperationException($"Invalid script role {activity.From.Role}."); + } + + if (testFailed) break; + } + } + + timer.Stop(); + + if (!testFailed) + { + logger.ForegroundColor($"Test script passed", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green); + } + + logger.Information($"Time: {timer.Elapsed.TotalSeconds.ToString("0.00")} seconds"); + + return !testFailed; + } + catch (Exception ex) + { + logger.ForegroundColor($"An error occurred while validating the chat transcript file. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red); + return false; + } + finally + { + WriteCSVLog(logRecords); + } + } + + /// + /// Receives an activityList from a transcript file + /// + /// + /// an activityList + private async Task GetActivitiesFromTranscriptFile(string fileName) + { + using var reader = new StreamReader(fileName); + var transcript = await reader.ReadToEndAsync().ConfigureAwait(false); + var activityList = JsonConvert.DeserializeObject(transcript); + + if (activityList.Activities.Count == 0 && activityList.list_of_conversations.Count == 0) + { + throw new JsonReaderException(); + } + else if (activityList.list_of_conversations.Count > 0) + { + foreach (var activities in activityList.list_of_conversations) + { + AddTextLineNumber(activities, fileName); + } + } + else + { + AddTextLineNumber(activityList, fileName); + } + + return activityList; + } + + /// + /// Gets the line number for the activity text + /// + /// + /// + private void AddTextLineNumber(ActivityList activityList, string fileName) + { + var lineNumber = 0; + var lines = File.ReadLines(fileName).ToList(); + + foreach (var activity in activityList.Activities.Where(a => a.IsMessageActivityWithText())) + { + // Added line number for Text messages + lineNumber = lines.FindIndex(lineNumber, line => line.Contains(activity.Text)) + 1; + activity.LineNumber = lineNumber; + } + } + + /// + /// Write CSV log + /// + /// + private void WriteCSVLog(List records) + { + var csvLogPath = Path.Combine(Environment.CurrentDirectory, "logFile.csv"); + var config = new CsvConfiguration(CultureInfo.InvariantCulture) + { + // Don't write the header again. + HasHeaderRecord = false, + // Use the system list separator + Delimiter = CultureInfo.CurrentCulture.TextInfo.ListSeparator + }; + + if (!File.Exists(csvLogPath)) + { + using (var streamWriter = new StreamWriter(csvLogPath)) + { + using (var csvWriter = new CsvWriter(streamWriter, config)) + { + csvWriter.WriteHeader(); + csvWriter.NextRecord(); + } + } + } + + using (var stream = File.Open(csvLogPath, FileMode.Append)) + { + using (var streamWriter = new StreamWriter(stream)) + { + using (var csvWriter = new CsvWriter(streamWriter, config)) + { + csvWriter.WriteRecords(records); + } + } + } + } + + /// + /// Check if adaptive cards structure are equal + /// + /// + /// + /// boolean + private bool AssertActivity(Models.Activities.Activity expectedActivity, Activity receivedActivity) + { + bool result = true; + if (!string.IsNullOrEmpty(expectedActivity.Text) && !string.IsNullOrEmpty(receivedActivity.Text)) + { + // Replace unwanted characters + receivedActivity.Text = receivedActivity.Text.Replace((char)0xA0, ' '); + + string pattern = ExtractRegex(expectedActivity.Text); + if (!string.IsNullOrEmpty(pattern)) + { + // If the line contains a regex pattern it will check if matches + return Regex.IsMatch(receivedActivity.Text, pattern); + } + // Allow for new lines in chat file, json and/or activity text by completely unescaping both strings - Issue 218 + else if (!expectedActivity.Text.CompletelyUnescape().Equals(receivedActivity.Text.CompletelyUnescape(), StringComparison.InvariantCultureIgnoreCase)) + { + // This is a simple text to compare + return false; + } + } + else if (expectedActivity.Attachments != null && expectedActivity.Attachments.Count == receivedActivity.Attachments.Count) + { + // This is an adaptive card, so the structure comparison will be executed + var expectedAttachments = expectedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); + var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList(); + var settings = new AdaptiveCardTranslatorSettings(); + + for (int i = 0; i < expectedActivity.Attachments.Count; i++) + { + var expectedCard = AdaptiveCard.GetCardWithoutValues(expectedAttachments[i].ToJObject(true), settings); + var receivedCard = AdaptiveCard.GetCardWithoutValues(receivedAttachments[i].ToJObject(true), settings); + if (!expectedCard.Equals(receivedCard, StringComparison.InvariantCultureIgnoreCase)) + { + return false; + } + } + } + else + { + return false; + } + + return result; + } + + /// + /// validate if the activity should be ignored + /// + /// + /// boolean + private bool IgnoreActivity(Models.Activities.Activity activity) + { + // Ignore trace activities unless it is an IntentCandidates type one. Also, ignore the DYM message as it is sent in the previous activity + return (activity.Type == Helpers.ActivityTypes.Trace + && !activity.ValueType.Equals("IntentCandidates", StringComparison.InvariantCultureIgnoreCase)) + || (activity.Type == Helpers.ActivityTypes.Message + && activity.Text != null + && activity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase)); + } + + /// + /// Extract the regex pattern + /// + /// + /// path + private string ExtractRegex(string input) + { + if (input == null) + { + return null; + } + string regexPattern = "<\\((.*?)\\)>"; // The regex pattern to search for + Regex regex = new Regex(regexPattern); + Match match = regex.Match(input); + if (match.Success) + { + return match.Groups[1].Value; // Extract the REGEX + } + else + { + return null; + } + } + + /// + /// Convert a datetime to Unix time format + /// + /// + /// path + private int ToUnixTimeSeconds(DateTime date) + { + DateTime point = new DateTime(1970, 1, 1); + TimeSpan time = date.Subtract(point); + + return (int)time.TotalSeconds; + } + } +} diff --git a/PVATestFramework/PVATestFramework/Models/Activities/Activity.cs b/PVATestFramework/PVATestFramework/Models/Activities/Activity.cs index bb5f0837..be148427 100644 --- a/PVATestFramework/PVATestFramework/Models/Activities/Activity.cs +++ b/PVATestFramework/PVATestFramework/Models/Activities/Activity.cs @@ -49,11 +49,16 @@ public static bool IsMessageActivityWithText(this Activity activity) public class Attachment { + public Attachment(string contentType, Object content) + { + ContentType = contentType; + Content = content; + } [JsonProperty("contentType")] public string ContentType { get; set; } [JsonProperty("content")] - public Content Content { get; set; } + public Object Content { get; set; } } public class Action @@ -121,6 +126,17 @@ public class Content [JsonProperty("version")] public string Version { get; set; } } + public class BasicCardContent + { + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("images")] + public List Images { get; set; } + + [JsonProperty("buttons")] + public List Buttons { get; set; } + } public class Data { diff --git a/PVATestFramework/PVATestFramework/PVATestFramework.sln b/PVATestFramework/PVATestFramework/PVATestFramework.sln new file mode 100644 index 00000000..c71b0b68 --- /dev/null +++ b/PVATestFramework/PVATestFramework/PVATestFramework.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PVATestFramework", "PVATestFramework.csproj", "{93399C26-6CC4-4C53-987B-166122B78300}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {93399C26-6CC4-4C53-987B-166122B78300}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {93399C26-6CC4-4C53-987B-166122B78300}.Debug|Any CPU.Build.0 = Debug|Any CPU + {93399C26-6CC4-4C53-987B-166122B78300}.Release|Any CPU.ActiveCfg = Release|Any CPU + {93399C26-6CC4-4C53-987B-166122B78300}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {C68EE9E9-3236-418C-8F2F-23E9420EA006} + EndGlobalSection +EndGlobal