diff --git a/PVATestFramework/PVATestFramework/Helpers/Constants.cs b/PVATestFramework/PVATestFramework/Helpers/Constants.cs
index 94a886d3..66cec173 100644
--- a/PVATestFramework/PVATestFramework/Helpers/Constants.cs
+++ b/PVATestFramework/PVATestFramework/Helpers/Constants.cs
@@ -16,6 +16,11 @@ public static class RoleTypes
public const int Bot = 0;
}
+ public static class Content
+ {
+ public const string BasicCard = "application/vnd.microsoft.card.hero";
+ }
+
public static class Constants
{
public const string DataverseTokenUri = "https://login.microsoftonline.com/{TenantId}/oauth2/v2.0/token";
diff --git a/PVATestFramework/PVATestFramework/Helpers/Runner.cs b/PVATestFramework/PVATestFramework/Helpers/Runner.cs
index 11eb94d8..4a9b37fc 100644
--- a/PVATestFramework/PVATestFramework/Helpers/Runner.cs
+++ b/PVATestFramework/PVATestFramework/Helpers/Runner.cs
@@ -1,813 +1,1544 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License.
-
-using PVATestFramework.Console.Helpers;
-using PVATestFramework.Console.Helpers.DirectLine;
-using PVATestFramework.Console.Helpers.Extensions;
-using PVATestFramework.Console.Helpers.FileHandler;
-using PVATestFramework.Console.Models;
-using PVATestFramework.Console.Models.Activities;
-using PVATestFramework.Console.Models.DirectLine;
-using CsvHelper;
-using CsvHelper.Configuration;
-using Newtonsoft.Json;
-using Serilog;
-using System.Diagnostics;
-using System.Globalization;
-using System.Text.RegularExpressions;
-using Activity = Microsoft.Bot.Connector.DirectLine.Activity;
-using LoggerExtensions = PVATestFramework.Console.Helpers.Extensions.LoggerExtensions;
-using Newtonsoft.Json.Linq;
-
-namespace PVATestFramework.Console
-{
- public class Runner
- {
- private readonly ILogger logger;
- private readonly IFileHandler fileHandler;
- private readonly DirectLineClientBase directLineClient;
-
- public Runner(ILogger logger, IFileHandler fileHandler)
- {
- this.logger = logger;
- this.fileHandler = fileHandler;
- }
- public Runner(ILogger logger, DirectLineClientBase directLineClient, IFileHandler fileHandler)
- {
- this.logger = logger;
- this.directLineClient = directLineClient;
- this.fileHandler = fileHandler;
- }
-
- ///
- /// Run a chat transcript test from a directory or file
- ///
- ///
- ///
- ///
- ///
- /// boolean depending if the execution was successful or not
- public async Task RunTranscriptTestAsync(DirectLineOptions options, string path, bool verbose = false, CancellationToken cancellationToken = default)
- {
- logger.Information("The test has started...");
-
- try
- {
- var totalTests = 0;
- var failedTests = 0;
- var fileAttributes = fileHandler.GetFileAttributes(path);
- var fileList = new List();
-
- if (fileAttributes.HasFlag(FileAttributes.Directory))
- {
- // The path is a directory
- fileList = fileHandler.GetFilesFromDirectory(path, "*.json").ToList();
- }
- else
- {
- // The path is just a file
- fileList.Add(path);
- }
-
- logger.Information($"Using Direct Line endpoint: {options.RegionalEndpoint}");
-
- foreach (var file in fileList)
- {
- logger.Information($"Testing with file: {file}");
-
- var activityList = await GetActivitiesFromTranscriptFile(file);
-
- if (activityList.list_of_conversations.Count > 0)
- {
- // The file has multiple conversations
- foreach (var actlist in activityList.list_of_conversations)
- {
- totalTests++;
- var result = await ExecuteTranscriptAsync(options, actlist, file, verbose, cancellationToken).ConfigureAwait(false);
- if (!result) failedTests++;
- }
- }
- else
- {
- // The file has a single conversation
- totalTests++;
- var result = await ExecuteTranscriptAsync(options, activityList, file, verbose, cancellationToken).ConfigureAwait(false);
- if (!result) failedTests++;
- }
- }
-
- logger.Information($"Test results: {totalTests - failedTests} passed, {totalTests} total");
-
- return failedTests == 0;
- }
- catch (JsonReaderException ex)
- {
- logger.ForegroundColor($"An error occurred while running the test. Details: The json file used as an input is not valid.", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
- return false;
- }
- catch (Exception ex)
- {
- logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
- return false;
- }
- }
-
- ///
- /// Run a scale test, sending N conversations to the bot
- ///
- ///
- ///
- ///
- ///
- /// boolean depending if the execution was successful or not
- public async Task RunScaleTestAsync(DirectLineOptions options, int totalAttempts, string path, bool verbose)
- {
- logger.Information($"Running a Scale test (sending chat transcript {totalAttempts} times).");
-
- try
- {
- var succeededAttempts = 0;
- for (int i = 0; i < totalAttempts; i++)
- {
- if (verbose)
- {
- logger.Information($"Attempt {i+1}...");
- }
- try
- {
- var result = await RunTranscriptTestAsync(options, path, verbose);
- if (result)
- {
- succeededAttempts++;
- }
- }
- catch
- {
- logger.ForegroundColor($"The chat transcript failed.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- }
- }
-
- if (succeededAttempts == totalAttempts)
- {
- logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
- logger.ForegroundColor($"Scale test ended succesfully.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
- return true;
- }
- else
- {
- logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts from {totalAttempts}.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- logger.ForegroundColor($"Scale test ended with errors.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- return false;
- }
- }
- catch (Exception ex)
- {
- logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
- return false;
- }
- }
-
- ///
- /// Convert a .chat file into a .json file
- ///
- ///
- ///
- /// boolean depending if the conversion was successful or not
- public bool ConvertChatFileToJSON(string path, string outputFile)
- {
- try
- {
- logger.Information($"The conversion process has started...");
-
- var line = string.Empty;
- var activityListContainer = new List>();
- var activityList = new List();
- string filepath = fileHandler.GetFullPath(outputFile);
-
- object? channelData = null;
-
- if (!filepath.EndsWith(".json"))
- {
- throw new ArgumentException("The output file should have a .json extension.");
- }
- if (!path.EndsWith(".chat"))
- {
- throw new ArgumentException("The input file should have a .chat extension.");
- }
-
- using (StreamReader sr = new StreamReader(path))
- {
- fileHandler.DeleteFile(filepath);
-
- while (null != (line = sr.ReadLine()))
- {
- var activity = new Models.Activities.Activity();
- if (string.IsNullOrWhiteSpace(line.Trim()))
- {
- continue;
- }
- else if (line.Trim().Equals("", StringComparison.InvariantCultureIgnoreCase))
- {
- activityListContainer.Add(activityList);
- activityList = new List();
- //reset channeldata for each conversation
- channelData = null;
- continue;
- }
- else if (line.StartsWith("user:"))
- {
- var userReg = new Regex(Regex.Escape("user:"));
- var userText = userReg.Replace(line, string.Empty, 1).Trim();
- if (string.IsNullOrEmpty(userText))
- {
- throw new ArgumentException("The user message is null or empty.");
- }
- else
- {
- activity = new Models.Activities.Activity
- {
- Type = Helpers.ActivityTypes.Message,
- Text = userText,
- From = new From(string.Empty, 1),
- Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
- ChannelData = channelData
- };
- }
- }
- else if (line.StartsWith("userEvent:"))
- {
- var userEventRegex = new Regex(Regex.Escape("userEvent:"));
- var userEventText = userEventRegex.Replace(line, string.Empty, 1).Trim();
-
- var userEventInfo = JObject.Parse(userEventText);
-
- if (string.IsNullOrEmpty(userEventText) || userEventInfo == null)
- {
- throw new ArgumentException("The userEvent is invalid");
- }
- else
- {
- activity = new Models.Activities.Activity
- {
- Type = ActivityTypes.Event,
- From = new From("user", 1),
- Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
- ChannelData = channelData,
- Name = (string)userEventInfo["Name"],
- Value = (string)userEventInfo["Value"]
- };
- }
- }
- else if (line.StartsWith("userMessage:"))
- {
- var userMessageRegex = new Regex(Regex.Escape("userMessage:"));
- var userMessageText = userMessageRegex.Replace(line, string.Empty, 1).Trim();
-
- var userMessageInfo = JObject.Parse(userMessageText);
-
- if (string.IsNullOrEmpty(userMessageText) || userMessageInfo == null)
- {
- throw new ArgumentException("The userMessage is invalid");
- }
- else
- {
- activity = new Models.Activities.Activity
- {
- Type = ActivityTypes.Message,
- From = new From("user", 1),
- Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
- ChannelData = channelData,
- Text = (string)userMessageInfo["Text"],
- Value = (string)userMessageInfo["Value"]
- };
- }
- }
- else if (line.StartsWith("bot:"))
- {
- var botReg = new Regex(Regex.Escape("bot:"));
- var botText = botReg.Replace(line, string.Empty, 1).Trim();
- if (string.IsNullOrEmpty(botText))
- {
- throw new ArgumentException("The bot message is null or empty.");
- }
- else
- {
- activity = new Models.Activities.Activity
- {
- Type = Helpers.ActivityTypes.Message,
- Text = botText,
- From = new From(string.Empty, 0),
- Timestamp = ToUnixTimeSeconds(DateTime.UtcNow)
- };
- }
- }
- else if (line.StartsWith("botEvent:"))
- {
- var botEventRegex = new Regex(Regex.Escape("botEvent:"));
- var botEventText = botEventRegex.Replace(line, string.Empty, 1).Trim();
-
- var botEventInfo = JObject.Parse(botEventText);
-
- if (string.IsNullOrEmpty(botEventText) || botEventInfo == null)
- {
- throw new ArgumentException("The botEvent is invalid");
- }
- else
- {
- activity = new Models.Activities.Activity
- {
- Type = ActivityTypes.Event,
- From = new From("bot", 0),
- Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
- ChannelData = channelData,
- Name = (string)botEventInfo["Name"],
- Value = (string)botEventInfo["Value"]
- };
- }
- }
- else if (line.StartsWith("suggested:"))
- {
- var suggestionRegex = new Regex(Regex.Escape("suggested:"));
- var suggestionText = suggestionRegex.Replace(line, string.Empty, 1).Trim();
-
- if (string.IsNullOrEmpty(suggestionText))
- {
- throw new ArgumentException("The suggested message is null or empty.");
- }
-
- var suggestionList = suggestionText.Split('|', StringSplitOptions.RemoveEmptyEntries).ToList();
- var intentCandidates = new List();
-
- foreach (var suggestion in suggestionList)
- {
- intentCandidates.Add(new IntentCandidate
- {
- IntentScore = new IntentScore()
- {
- Title = suggestion
- }
- });
- }
- activity = new Models.Activities.Activity
- {
- ValueType = "IntentCandidates",
- Type = Helpers.ActivityTypes.Trace,
- From = new From(string.Empty, 0),
- Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
- Value = new Value()
- {
- IntentCandidates = intentCandidates
- }
- };
- }
- else if (line.StartsWith("channelData:"))
- {
- var channelDataRegex = new Regex("channelData:");
- var channelDataText = channelDataRegex.Replace(line, string.Empty, 1).Trim();
-
- if (string.IsNullOrEmpty(channelDataText))
- {
- channelData = null;
- continue;
- }
- else
- {
- channelData = JObject.Parse(channelDataText);
-
- // if this is the first thing in the conversation (nothing in the activity list yet), add a startConversation event
- if(activityList.Count == 0)
- {
- activity = new Models.Activities.Activity
- {
- Type = Helpers.ActivityTypes.Event,
- From = new From(string.Empty, 1),
- ChannelData = channelData,
- Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
- Name = "StartConversation"
- };
- }
- else
- {
- continue;
- }
- }
- }
- else
- {
- throw new Exception("The input file format is not valid.");
- }
-
- activityList.Add(activity);
- }
- activityListContainer.Add(activityList);
-
- var activities = new List();
- foreach (var list in activityListContainer)
- {
- activities.Add($"{{\"Activities\":{string.Join(',', JsonConvert.SerializeObject(list))}}}");
- }
- fileHandler.WriteToFile(filepath, $"{{\"list_of_conversations\":[{string.Join(",", activities)}]}}");
- }
-
- logger.Information($"The conversion process ended.");
- return true;
- }
- catch (Exception ex)
- {
- logger.ForegroundColor($"An error occurred while converting the .chat transcript. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
- return false;
- }
- }
-
- ///
- /// Execute the conversation on the transcript
- ///
- ///
- ///
- ///
- ///
- ///
- /// boolean depending if the conversion was successful or not
- ///
- private async Task ExecuteTranscriptAsync(DirectLineOptions options, ActivityList activityList, string path, bool verbose = false, CancellationToken cancellationToken = default)
- {
- var logRecords = new List();
- var activities = new List();
- var timer = new Stopwatch();
- var testFailed = false;
- var userUtterance = string.Empty;
-
- timer.Start();
- path = fileHandler.GetFullPath(path);
-
- try
- {
- using (directLineClient)
- {
- foreach (var activity in activityList.Activities)
- {
- switch (activity.From.Role)
- {
- case RoleTypes.User:
- var sendActivity = new Activity
- {
- Type = activity.Type,
- Text = activity.Text,
- ChannelData = activity.ChannelData,
- Name= activity.Name,
- Value = activity.Value
- };
-
- userUtterance = sendActivity.Text;
- if (verbose)
- {
- logger.Information($"User sends: {sendActivity.Text}");
- }
-
- await directLineClient.SendActivityAsync(sendActivity, cancellationToken).ConfigureAwait(false);
- break;
-
- case RoleTypes.Bot:
- if (IgnoreActivity(activity))
- {
- break;
- }
-
- var receivedActivity = new Activity();
- var receivedOptions = new List();
- var expectedOptions = new List();
-
- if (activities.Count == 0)
- {
- activities = await directLineClient.ReceiveActivitiesAsync(cancellationToken).ConfigureAwait(false);
-
- // Get the first activity from the bot response
- receivedActivity = activities.FirstOrDefault();
- activities.Remove(receivedActivity);
-
- if (verbose)
- {
- logger.Information($"Bot sends: {receivedActivity.Text}");
- }
- if (receivedActivity.SuggestedActions != null)
- {
- // Get the suggested topics from the activity if any
- receivedOptions = receivedActivity.SuggestedActions?.Actions?.Where(o => !o.Title.Equals(BotDefaultMessages.NoneOfThese, StringComparison.InvariantCultureIgnoreCase)).Select(a => a.Title).ToList();
-
- if (verbose)
- {
- logger.Information($"\t{string.Join(" | ", receivedOptions)}");
- }
- }
- }
- else
- {
- // Get the first activity from the list
- receivedActivity = activities.FirstOrDefault();
- activities.Remove(receivedActivity);
-
- if (verbose)
- {
- logger.Information($"Bot sends: {receivedActivity.Text}");
- }
- }
-
- var csvRecord = new LogCSV()
- {
- BotId = receivedActivity.From.Id,
- ConversationId = receivedActivity.Conversation.Id,
- SessionDate = DateTime.Now.ToString(),
- UserUtterance = userUtterance,
- ExpectedResponse = activity.Text,
- ReceivedResponse = receivedActivity.Text,
- TestFile = path
- };
-
- if (receivedActivity.Text != null && receivedActivity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase))
- {
- // activity.Value redefined as type object?, so cast it back to type Value for this particular situation
- expectedOptions = ((Value)(activity.Value))?.IntentCandidates != null ? ((Value)(activity.Value))?.IntentCandidates?.Select(o => o.IntentScore.Title).ToList() : new List() { "No suggested topics found" };
-
- for (int i = 0; i < receivedOptions.Count; i++)
- {
- if (i == 0) csvRecord.DYM_Option1 = receivedOptions[i];
- if (i == 1) csvRecord.DYM_Option2 = receivedOptions[i];
- if (i == 2) csvRecord.DYM_Option3 = receivedOptions[i];
- }
-
- if (expectedOptions.Count != receivedOptions.Count)
- {
- testFailed = true;
- }
- else
- {
- foreach (var suggestion in receivedOptions)
- {
- if (!expectedOptions.Any(option => option.Equals(suggestion, StringComparison.InvariantCultureIgnoreCase)))
- {
- testFailed = true;
- break;
- }
- }
- }
-
- if (testFailed)
- {
- logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- logger.ForegroundColor($"Expected:\t{string.Join(" | ", expectedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- logger.ForegroundColor($"Received:\t{string.Join(" | ", receivedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- }
- }
- else
- {
- if (!AssertActivity(activity, receivedActivity))
- {
- logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
-
- if (!string.IsNullOrEmpty(activity.Text))
- {
- logger.ForegroundColor($"Expected: {activity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- logger.ForegroundColor($"Received: {receivedActivity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- logger.ForegroundColor($"Line number: {activity.LineNumber}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- }
-
- if (activity.Attachments != null && activity.Attachments.Count > 0)
- {
- var expectedAttachments = activity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
- var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
-
- logger.ForegroundColor($"Expected: {string.Join(Environment.NewLine, expectedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- logger.ForegroundColor($"Received: {string.Join(Environment.NewLine, receivedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
- }
-
- testFailed = true;
- }
- }
-
- userUtterance = string.Empty;
- csvRecord.Result = testFailed ? "Failed" : "Passed";
- logRecords.Add(csvRecord);
- break;
-
- default:
- throw new InvalidOperationException($"Invalid script role {activity.From.Role}.");
- }
-
- if (testFailed) break;
- }
- }
-
- timer.Stop();
-
- if (!testFailed)
- {
- logger.ForegroundColor($"Test script passed", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
- }
-
- logger.Information($"Time: {timer.Elapsed.TotalSeconds.ToString("0.00")} seconds");
-
- return !testFailed;
- }
- catch (Exception ex)
- {
- logger.ForegroundColor($"An error occurred while validating the chat transcript file. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
- return false;
- }
- finally
- {
- WriteCSVLog(logRecords);
- }
- }
-
- ///
- /// Receives an activityList from a transcript file
- ///
- ///
- /// an activityList
- private async Task GetActivitiesFromTranscriptFile(string fileName)
- {
- using var reader = new StreamReader(fileName);
- var transcript = await reader.ReadToEndAsync().ConfigureAwait(false);
- var activityList = JsonConvert.DeserializeObject(transcript);
-
- if (activityList.Activities.Count == 0 && activityList.list_of_conversations.Count == 0)
- {
- throw new JsonReaderException();
- }
- else if (activityList.list_of_conversations.Count > 0)
- {
- foreach (var activities in activityList.list_of_conversations)
- {
- AddTextLineNumber(activities, fileName);
- }
- }
- else
- {
- AddTextLineNumber(activityList, fileName);
- }
-
- return activityList;
- }
-
- ///
- /// Gets the line number for the activity text
- ///
- ///
- ///
- private void AddTextLineNumber(ActivityList activityList, string fileName)
- {
- var lineNumber = 0;
- var lines = File.ReadLines(fileName).ToList();
-
- foreach (var activity in activityList.Activities.Where(a => a.IsMessageActivityWithText()))
- {
- // Added line number for Text messages
- lineNumber = lines.FindIndex(lineNumber, line => line.Contains(activity.Text)) + 1;
- activity.LineNumber = lineNumber;
- }
- }
-
- ///
- /// Write CSV log
- ///
- ///
- private void WriteCSVLog(List records)
- {
- var csvLogPath = Path.Combine(Environment.CurrentDirectory, "logFile.csv");
- var config = new CsvConfiguration(CultureInfo.InvariantCulture)
- {
- // Don't write the header again.
- HasHeaderRecord = false,
- // Use the system list separator
- Delimiter = CultureInfo.CurrentCulture.TextInfo.ListSeparator
- };
-
- if (!File.Exists(csvLogPath))
- {
- using (var streamWriter = new StreamWriter(csvLogPath))
- {
- using (var csvWriter = new CsvWriter(streamWriter, config))
- {
- csvWriter.WriteHeader();
- csvWriter.NextRecord();
- }
- }
- }
-
- using (var stream = File.Open(csvLogPath, FileMode.Append))
- {
- using (var streamWriter = new StreamWriter(stream))
- {
- using (var csvWriter = new CsvWriter(streamWriter, config))
- {
- csvWriter.WriteRecords(records);
- }
- }
- }
- }
-
- ///
- /// Check if adaptive cards structure are equal
- ///
- ///
- ///
- /// boolean
- private bool AssertActivity(Models.Activities.Activity expectedActivity, Activity receivedActivity)
- {
- bool result = true;
- if (!string.IsNullOrEmpty(expectedActivity.Text) && !string.IsNullOrEmpty(receivedActivity.Text))
- {
- // Replace unwanted characters
- receivedActivity.Text = receivedActivity.Text.Replace((char)0xA0, ' ');
-
- string pattern = ExtractRegex(expectedActivity.Text);
- if (!string.IsNullOrEmpty(pattern))
- {
- // If the line contains a regex pattern it will check if matches
- return Regex.IsMatch(receivedActivity.Text, pattern);
- }
- // Allow for new lines in chat file, json and/or activity text by completely unescaping both strings - Issue 218
- else if (!expectedActivity.Text.CompletelyUnescape().Equals(receivedActivity.Text.CompletelyUnescape(), StringComparison.InvariantCultureIgnoreCase))
- {
- // This is a simple text to compare
- return false;
- }
- }
- else if (expectedActivity.Attachments != null && expectedActivity.Attachments.Count == receivedActivity.Attachments.Count)
- {
- // This is an adaptive card, so the structure comparison will be executed
- var expectedAttachments = expectedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
- var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
- var settings = new AdaptiveCardTranslatorSettings();
-
- for (int i = 0; i < expectedActivity.Attachments.Count; i++)
- {
- var expectedCard = AdaptiveCard.GetCardWithoutValues(expectedAttachments[i].ToJObject(true), settings);
- var receivedCard = AdaptiveCard.GetCardWithoutValues(receivedAttachments[i].ToJObject(true), settings);
- if (!expectedCard.Equals(receivedCard, StringComparison.InvariantCultureIgnoreCase))
- {
- return false;
- }
- }
- }
- else
- {
- return false;
- }
-
- return result;
- }
-
- ///
- /// validate if the activity should be ignored
- ///
- ///
- /// boolean
- private bool IgnoreActivity(Models.Activities.Activity activity)
- {
- // Ignore trace activities unless it is an IntentCandidates type one. Also, ignore the DYM message as it is sent in the previous activity
- return (activity.Type == Helpers.ActivityTypes.Trace
- && !activity.ValueType.Equals("IntentCandidates", StringComparison.InvariantCultureIgnoreCase))
- || (activity.Type == Helpers.ActivityTypes.Message
- && activity.Text != null
- && activity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase));
- }
-
- ///
- /// Extract the regex pattern
- ///
- ///
- /// path
- private string ExtractRegex(string input)
- {
- if (input == null)
- {
- return null;
- }
- string regexPattern = "<\\((.*?)\\)>"; // The regex pattern to search for
- Regex regex = new Regex(regexPattern);
- Match match = regex.Match(input);
- if (match.Success)
- {
- return match.Groups[1].Value; // Extract the REGEX
- }
- else
- {
- return null;
- }
- }
-
- ///
- /// Convert a datetime to Unix time format
- ///
- ///
- /// path
- private int ToUnixTimeSeconds(DateTime date)
- {
- DateTime point = new DateTime(1970, 1, 1);
- TimeSpan time = date.Subtract(point);
-
- return (int)time.TotalSeconds;
- }
- }
-}
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using PVATestFramework.Console.Helpers;
+using PVATestFramework.Console.Helpers.DirectLine;
+using PVATestFramework.Console.Helpers.Extensions;
+using PVATestFramework.Console.Helpers.FileHandler;
+using PVATestFramework.Console.Models;
+using PVATestFramework.Console.Models.Activities;
+using PVATestFramework.Console.Models.DirectLine;
+using CsvHelper;
+using CsvHelper.Configuration;
+using Newtonsoft.Json;
+using Serilog;
+using System.Diagnostics;
+using System.Globalization;
+using System.Text.RegularExpressions;
+using Activity = Microsoft.Bot.Connector.DirectLine.Activity;
+using LoggerExtensions = PVATestFramework.Console.Helpers.Extensions.LoggerExtensions;
+
+namespace PVATestFramework.Console
+{
+ public class Runner
+ {
+ private readonly ILogger logger;
+ private readonly IFileHandler fileHandler;
+ private readonly DirectLineClientBase directLineClient;
+
+ public Runner(ILogger logger, IFileHandler fileHandler)
+ {
+ this.logger = logger;
+ this.fileHandler = fileHandler;
+ }
+ public Runner(ILogger logger, DirectLineClientBase directLineClient, IFileHandler fileHandler)
+ {
+ this.logger = logger;
+ this.directLineClient = directLineClient;
+ this.fileHandler = fileHandler;
+ }
+
+ ///
+ /// Run a chat transcript test from a directory or file
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// boolean depending if the execution was successful or not
+ public async Task RunTranscriptTestAsync(DirectLineOptions options, string path, bool verbose = false, CancellationToken cancellationToken = default)
+ {
+ logger.Information("The test has started...");
+
+ try
+ {
+ var totalTests = 0;
+ var failedTests = 0;
+ var fileAttributes = fileHandler.GetFileAttributes(path);
+ var fileList = new List();
+
+ if (fileAttributes.HasFlag(FileAttributes.Directory))
+ {
+ // The path is a directory
+ fileList = fileHandler.GetFilesFromDirectory(path, "*.json").ToList();
+ }
+ else
+ {
+ // The path is just a file
+ fileList.Add(path);
+ }
+
+ logger.Information($"Using Direct Line endpoint: {options.RegionalEndpoint}");
+
+ foreach (var file in fileList)
+ {
+ logger.Information($"Testing with file: {file}");
+
+ var activityList = await GetActivitiesFromTranscriptFile(file);
+
+ if (activityList.list_of_conversations.Count > 0)
+ {
+ // The file has multiple conversations
+ foreach (var actlist in activityList.list_of_conversations)
+ {
+ totalTests++;
+ var result = await ExecuteTranscriptAsync(options, actlist, file, verbose, cancellationToken).ConfigureAwait(false);
+ if (!result) failedTests++;
+ }
+ }
+ else
+ {
+ // The file has a single conversation
+ totalTests++;
+ var result = await ExecuteTranscriptAsync(options, activityList, file, verbose, cancellationToken).ConfigureAwait(false);
+ if (!result) failedTests++;
+ }
+ }
+
+ logger.Information($"Test results: {totalTests - failedTests} passed, {totalTests} total");
+
+ return failedTests == 0;
+ }
+ catch (JsonReaderException ex)
+ {
+ logger.ForegroundColor($"An error occurred while running the test. Details: The json file used as an input is not valid.", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ catch (Exception ex)
+ {
+ logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ }
+
+ ///
+ /// Run a scale test, sending N conversations to the bot
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// boolean depending if the execution was successful or not
+ public async Task RunScaleTestAsync(DirectLineOptions options, int totalAttempts, string path, bool verbose)
+ {
+ logger.Information($"Running a Scale test (sending chat transcript {totalAttempts} times).");
+
+ try
+ {
+ var succeededAttempts = 0;
+ for (int i = 0; i < totalAttempts; i++)
+ {
+ if (verbose)
+ {
+ logger.Information($"Attempt {i+1}...");
+ }
+ try
+ {
+ var result = await RunTranscriptTestAsync(options, path, verbose);
+ if (result)
+ {
+ succeededAttempts++;
+ }
+ }
+ catch
+ {
+ logger.ForegroundColor($"The chat transcript failed.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ }
+ }
+
+ if (succeededAttempts == totalAttempts)
+ {
+ logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
+ logger.ForegroundColor($"Scale test ended succesfully.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
+ return true;
+ }
+ else
+ {
+ logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts from {totalAttempts}.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Scale test ended with errors.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ return false;
+ }
+ }
+ catch (Exception ex)
+ {
+ logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ }
+
+ ///
+ /// Convert a .chat file into a .json file
+ ///
+ ///
+ ///
+ /// boolean depending if the conversion was successful or not
+ public bool ConvertChatFileToJSON(string path, string outputFile)
+ {
+ try
+ {
+ logger.Information($"The conversion process has started...");
+
+ var line = string.Empty;
+ var activityListContainer = new List>();
+ var activityList = new List();
+ string filepath = fileHandler.GetFullPath(outputFile);
+
+ if (!filepath.EndsWith(".json"))
+ {
+ throw new ArgumentException("The output file should have a .json extension.");
+ }
+ if (!path.EndsWith(".chat"))
+ {
+ throw new ArgumentException("The input file should have a .chat extension.");
+ }
+
+ using (StreamReader sr = new StreamReader(path))
+ {
+ fileHandler.DeleteFile(filepath);
+
+ while (null != (line = sr.ReadLine()))
+ {
+ var activity = new Models.Activities.Activity();
+ if (string.IsNullOrWhiteSpace(line.Trim()))
+ {
+ continue;
+ }
+ else if (line.Trim().Equals("", StringComparison.InvariantCultureIgnoreCase))
+ {
+ activityListContainer.Add(activityList);
+ activityList = new List();
+ continue;
+ }
+ else if (line.StartsWith("user:"))
+ {
+ var userReg = new Regex(Regex.Escape("user:"));
+ var userText = userReg.Replace(line, string.Empty, 1).Trim();
+ if (string.IsNullOrEmpty(userText))
+ {
+ throw new ArgumentException("The user message is null or empty.");
+ }
+ else
+ {
+ activity = new Models.Activities.Activity
+ {
+ Type = Helpers.ActivityTypes.Message,
+ Text = userText,
+ From = new From(string.Empty, 1),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow)
+ };
+ }
+ }
+ else if (line.StartsWith("bot:"))
+ {
+ var botReg = new Regex(Regex.Escape("bot:"));
+ var botText = botReg.Replace(line, string.Empty, 1).Trim();
+ if (string.IsNullOrEmpty(botText))
+ {
+ throw new ArgumentException("The bot message is null or empty.");
+ }
+ else
+ {
+ activity = new Models.Activities.Activity
+ {
+ Type = Helpers.ActivityTypes.Message,
+ Text = botText,
+ From = new From(string.Empty, 0),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow)
+ };
+ }
+ }
+ else if (line.StartsWith("suggested:"))
+ {
+ var suggestionRegex = new Regex(Regex.Escape("suggested:"));
+ var suggestionText = suggestionRegex.Replace(line, string.Empty, 1).Trim();
+
+ if (string.IsNullOrEmpty(suggestionText))
+ {
+ throw new ArgumentException("The suggested message is null or empty.");
+ }
+
+ var suggestionList = suggestionText.Split('|', StringSplitOptions.RemoveEmptyEntries).ToList();
+ var intentCandidates = new List();
+
+ foreach (var suggestion in suggestionList)
+ {
+ intentCandidates.Add(new IntentCandidate
+ {
+ IntentScore = new IntentScore()
+ {
+ Title = suggestion
+ }
+ });
+ }
+ activity = new Models.Activities.Activity
+ {
+ ValueType = "IntentCandidates",
+ Type = Helpers.ActivityTypes.Trace,
+ From = new From(string.Empty, 0),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
+ Value = new Value()
+ {
+ IntentCandidates = intentCandidates
+ }
+ };
+ }
+ else if (line.StartsWith("attachment:"))
+ {
+ var attachmentRegex = new Regex(Regex.Escape("attachment:"));
+ var attachmentTitle = attachmentRegex.Replace(line, string.Empty, 1).Trim();
+ if (string.IsNullOrEmpty(attachmentTitle)){
+ throw new ArgumentException("The attachment title is null or empty.");
+ }
+ else {
+ var lastActivityList = activityList.Last();
+ lastActivityList.Attachments = [new Attachment(Helpers.Content.BasicCard, new BasicCardContent
+ {
+ Title = attachmentTitle,
+ Images = new List(),
+ Buttons = new List()
+ })];
+ continue;
+ }
+ }
+ else
+ {
+ throw new Exception("The input file format is not valid.");
+ }
+ activityList.Add(activity);
+ }
+ activityListContainer.Add(activityList);
+
+ var activities = new List();
+ foreach (var list in activityListContainer)
+ {
+ activities.Add($"{{\"Activities\":{string.Join(',', JsonConvert.SerializeObject(list))}}}");
+ }
+ fileHandler.WriteToFile(filepath, $"{{\"list_of_conversations\":[{string.Join(",", activities)}]}}");
+ }
+
+ logger.Information($"The conversion process ended.");
+ return true;
+ }
+ catch (Exception ex)
+ {
+ logger.ForegroundColor($"An error occurred while converting the .chat transcript. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ }
+
+ ///
+ /// Execute the conversation on the transcript
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// boolean depending if the conversion was successful or not
+ ///
+ private async Task ExecuteTranscriptAsync(DirectLineOptions options, ActivityList activityList, string path, bool verbose = false, CancellationToken cancellationToken = default)
+ {
+ var logRecords = new List();
+ var activities = new List();
+ var timer = new Stopwatch();
+ var testFailed = false;
+ var userUtterance = string.Empty;
+
+ timer.Start();
+ path = fileHandler.GetFullPath(path);
+
+ try
+ {
+ using (directLineClient)
+ {
+ foreach (var activity in activityList.Activities)
+ {
+ switch (activity.From.Role)
+ {
+ case RoleTypes.User:
+ var sendActivity = new Activity
+ {
+ Type = activity.Type,
+ Text = activity.Text
+ };
+
+ userUtterance = sendActivity.Text;
+ if (verbose)
+ {
+ logger.Information($"User sends: {sendActivity.Text}");
+ }
+
+ await directLineClient.SendActivityAsync(sendActivity, cancellationToken).ConfigureAwait(false);
+ break;
+
+ case RoleTypes.Bot:
+ if (IgnoreActivity(activity))
+ {
+ break;
+ }
+
+ var receivedActivity = new Activity();
+ var receivedOptions = new List();
+ var expectedOptions = new List();
+
+ if (activities.Count == 0)
+ {
+ activities = await directLineClient.ReceiveActivitiesAsync(cancellationToken).ConfigureAwait(false);
+
+ // Get the first activity from the bot response
+ receivedActivity = activities.FirstOrDefault();
+ activities.Remove(receivedActivity);
+
+ if (verbose)
+ {
+ logger.Information($"Bot sends: {receivedActivity.Text}");
+ }
+ if (receivedActivity.SuggestedActions != null)
+ {
+ // Get the suggested topics from the activity if any
+ receivedOptions = receivedActivity.SuggestedActions?.Actions?.Where(o => !o.Title.Equals(BotDefaultMessages.NoneOfThese, StringComparison.InvariantCultureIgnoreCase)).Select(a => a.Title).ToList();
+
+ if (verbose)
+ {
+ logger.Information($"\t{string.Join(" | ", receivedOptions)}");
+ }
+ }
+ }
+ else
+ {
+ // Get the first activity from the list
+ receivedActivity = activities.FirstOrDefault();
+ activities.Remove(receivedActivity);
+
+ if (verbose)
+ {
+ logger.Information($"Bot sends: {receivedActivity.Text}");
+ }
+ }
+
+ var csvRecord = new LogCSV()
+ {
+ BotId = receivedActivity.From.Id,
+ ConversationId = receivedActivity.Conversation.Id,
+ SessionDate = DateTime.Now.ToString(),
+ UserUtterance = userUtterance,
+ ExpectedResponse = activity.Text,
+ ReceivedResponse = receivedActivity.Text,
+ TestFile = path
+ };
+
+ if (receivedActivity.Text != null && receivedActivity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase))
+ {
+ expectedOptions = activity.Value?.IntentCandidates != null ? activity.Value?.IntentCandidates?.Select(o => o.IntentScore.Title).ToList() : new List() { "No suggested topics found" };
+
+ for (int i = 0; i < receivedOptions.Count; i++)
+ {
+ if (i == 0) csvRecord.DYM_Option1 = receivedOptions[i];
+ if (i == 1) csvRecord.DYM_Option2 = receivedOptions[i];
+ if (i == 2) csvRecord.DYM_Option3 = receivedOptions[i];
+ }
+
+ if (expectedOptions.Count != receivedOptions.Count)
+ {
+ testFailed = true;
+ }
+ else
+ {
+ foreach (var suggestion in receivedOptions)
+ {
+ if (!expectedOptions.Any(option => option.Equals(suggestion, StringComparison.InvariantCultureIgnoreCase)))
+ {
+ testFailed = true;
+ break;
+ }
+ }
+ }
+
+ if (testFailed)
+ {
+ logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Expected:\t{string.Join(" | ", expectedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Received:\t{string.Join(" | ", receivedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ }
+ }
+ else
+ {
+ if (!AssertActivity(activity, receivedActivity))
+ {
+ logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+
+ if (!string.IsNullOrEmpty(activity.Text))
+ {
+ logger.ForegroundColor($"Expected: {activity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Received: {receivedActivity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Line number: {activity.LineNumber}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ }
+
+
+ testFailed = true;
+ }
+ if (activity.Attachments != null && activity.Attachments.Count > 0 && !AssertActivityAttachment(activity, receivedActivity))
+ {
+ logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ var expectedAttachments = activity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
+ var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
+ logger.ForegroundColor($"Expected: {string.Join(Environment.NewLine, expectedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Received: {string.Join(Environment.NewLine, receivedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ testFailed = true;
+ }
+ }
+ userUtterance = string.Empty;
+ csvRecord.Result = testFailed ? "Failed" : "Passed";
+ logRecords.Add(csvRecord);
+ break;
+
+ default:
+ throw new InvalidOperationException($"Invalid script role {activity.From.Role}.");
+ }
+
+ if (testFailed) break;
+ }
+ }
+
+ timer.Stop();
+
+ if (!testFailed)
+ {
+ logger.ForegroundColor($"Test script passed", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
+ }
+
+ logger.Information($"Time: {timer.Elapsed.TotalSeconds.ToString("0.00")} seconds");
+
+ return !testFailed;
+ }
+ catch (Exception ex)
+ {
+ logger.ForegroundColor($"An error occurred while validating the chat transcript file. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ finally
+ {
+ WriteCSVLog(logRecords);
+ }
+ }
+
+ ///
+ /// Receives an activityList from a transcript file
+ ///
+ ///
+ /// an activityList
+ private async Task GetActivitiesFromTranscriptFile(string fileName)
+ {
+ using var reader = new StreamReader(fileName);
+ var transcript = await reader.ReadToEndAsync().ConfigureAwait(false);
+ var activityList = JsonConvert.DeserializeObject(transcript);
+
+ if (activityList.Activities.Count == 0 && activityList.list_of_conversations.Count == 0)
+ {
+ throw new JsonReaderException();
+ }
+ else if (activityList.list_of_conversations.Count > 0)
+ {
+ foreach (var activities in activityList.list_of_conversations)
+ {
+ AddTextLineNumber(activities, fileName);
+ }
+ }
+ else
+ {
+ AddTextLineNumber(activityList, fileName);
+ }
+
+ return activityList;
+ }
+
+ ///
+ /// Gets the line number for the activity text
+ ///
+ ///
+ ///
+ private void AddTextLineNumber(ActivityList activityList, string fileName)
+ {
+ var lineNumber = 0;
+ var lines = File.ReadLines(fileName).ToList();
+
+ foreach (var activity in activityList.Activities.Where(a => a.IsMessageActivityWithText()))
+ {
+ // Added line number for Text messages
+ lineNumber = lines.FindIndex(lineNumber, line => line.Contains(activity.Text)) + 1;
+ activity.LineNumber = lineNumber;
+ }
+ }
+
+ ///
+ /// Write CSV log
+ ///
+ ///
+ private void WriteCSVLog(List records)
+ {
+ var csvLogPath = Path.Combine(Environment.CurrentDirectory, "logFile.csv");
+ var config = new CsvConfiguration(CultureInfo.InvariantCulture)
+ {
+ // Don't write the header again.
+ HasHeaderRecord = false,
+ // Use the system list separator
+ Delimiter = CultureInfo.CurrentCulture.TextInfo.ListSeparator
+ };
+
+ if (!File.Exists(csvLogPath))
+ {
+ using (var streamWriter = new StreamWriter(csvLogPath))
+ {
+ using (var csvWriter = new CsvWriter(streamWriter, config))
+ {
+ csvWriter.WriteHeader();
+ csvWriter.NextRecord();
+ }
+ }
+ }
+
+ using (var stream = File.Open(csvLogPath, FileMode.Append))
+ {
+ using (var streamWriter = new StreamWriter(stream))
+ {
+ using (var csvWriter = new CsvWriter(streamWriter, config))
+ {
+ csvWriter.WriteRecords(records);
+ }
+ }
+ }
+ }
+
+ ///
+ /// Check if adaptive cards structure are equal
+ ///
+ ///
+ ///
+ /// boolean
+ private bool AssertActivity(Models.Activities.Activity expectedActivity, Activity receivedActivity)
+ {
+ bool result = true;
+ if (!string.IsNullOrEmpty(expectedActivity.Text) && !string.IsNullOrEmpty(receivedActivity.Text))
+ {
+ // Replace unwanted characters
+ receivedActivity.Text = receivedActivity.Text.Replace((char)0xA0, ' ');
+
+ string pattern = ExtractRegex(expectedActivity.Text);
+ if (!string.IsNullOrEmpty(pattern))
+ {
+ // If the line contains a regex pattern it will check if matches
+ return Regex.IsMatch(receivedActivity.Text, pattern);
+ }
+ // Allow for new lines in chat file, json and/or activity text by completely unescaping both strings - Issue 218
+ else if (!expectedActivity.Text.CompletelyUnescape().Equals(receivedActivity.Text.CompletelyUnescape(), StringComparison.InvariantCultureIgnoreCase))
+ {
+ // This is a simple text to compare
+ return false;
+ }
+ }
+ else
+ {
+ return false;
+ }
+ return result;
+ }
+
+ private bool AssertActivityAttachment(Models.Activities.Activity expectedActivity, Activity receivedActivity)
+ {
+ bool result = true;
+ if (expectedActivity.Attachments != null && expectedActivity.Attachments.Count == receivedActivity.Attachments.Count)
+ {
+ // This is an adaptive card, so the structure comparison will be executed
+ var expectedAttachments = expectedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
+ var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
+ var settings = new AdaptiveCardTranslatorSettings();
+ for (int i = 0; i < expectedActivity.Attachments.Count; i++)
+ {
+ var expectedCard = AdaptiveCard.GetCardWithoutValues(expectedAttachments[i].ToJObject(true), settings);
+ var receivedCard = AdaptiveCard.GetCardWithoutValues(receivedAttachments[i].ToJObject(true), settings);
+ logger.Information($"expected card: {expectedAttachments[i]}");
+ logger.Information($"received card: {receivedAttachments[i]}");
+ if (!expectedCard.Equals(receivedCard, StringComparison.InvariantCultureIgnoreCase))
+ {
+ return false;
+ }
+ if (!expectedAttachments[i].Equals(receivedAttachments[i], StringComparison.InvariantCultureIgnoreCase) && expectedActivity.Attachments[i].ContentType == Helpers.Content.BasicCard)
+ {
+ return false;
+ }
+ }
+ }
+ else
+ {
+ return false;
+ }
+ return result;
+ }
+
+ ///
+ /// validate if the activity should be ignored
+ ///
+ ///
+ /// boolean
+ private bool IgnoreActivity(Models.Activities.Activity activity)
+ {
+ // Ignore trace activities unless it is an IntentCandidates type one. Also, ignore the DYM message as it is sent in the previous activity
+ return (activity.Type == Helpers.ActivityTypes.Trace
+ && !activity.ValueType.Equals("IntentCandidates", StringComparison.InvariantCultureIgnoreCase))
+ || (activity.Type == Helpers.ActivityTypes.Message
+ && activity.Text != null
+ && activity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase));
+ }
+
+ ///
+ /// Extract the regex pattern
+ ///
+ ///
+ /// path
+ private string ExtractRegex(string input)
+ {
+ if (input == null)
+ {
+ return null;
+ }
+ string regexPattern = "<\\((.*?)\\)>"; // The regex pattern to search for
+ Regex regex = new Regex(regexPattern);
+ Match match = regex.Match(input);
+ if (match.Success)
+ {
+ return match.Groups[1].Value; // Extract the REGEX
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+ ///
+ /// Convert a datetime to Unix time format
+ ///
+ ///
+ /// path
+ private int ToUnixTimeSeconds(DateTime date)
+ {
+ DateTime point = new DateTime(1970, 1, 1);
+ TimeSpan time = date.Subtract(point);
+
+ return (int)time.TotalSeconds;
+ }
+ }
+}
+=======
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using PVATestFramework.Console.Helpers;
+using PVATestFramework.Console.Helpers.DirectLine;
+using PVATestFramework.Console.Helpers.Extensions;
+using PVATestFramework.Console.Helpers.FileHandler;
+using PVATestFramework.Console.Models;
+using PVATestFramework.Console.Models.Activities;
+using PVATestFramework.Console.Models.DirectLine;
+using CsvHelper;
+using CsvHelper.Configuration;
+using Newtonsoft.Json;
+using Serilog;
+using System.Diagnostics;
+using System.Globalization;
+using System.Text.RegularExpressions;
+using Activity = Microsoft.Bot.Connector.DirectLine.Activity;
+using LoggerExtensions = PVATestFramework.Console.Helpers.Extensions.LoggerExtensions;
+using Newtonsoft.Json.Linq;
+
+namespace PVATestFramework.Console
+{
+ public class Runner
+ {
+ private readonly ILogger logger;
+ private readonly IFileHandler fileHandler;
+ private readonly DirectLineClientBase directLineClient;
+
+ public Runner(ILogger logger, IFileHandler fileHandler)
+ {
+ this.logger = logger;
+ this.fileHandler = fileHandler;
+ }
+ public Runner(ILogger logger, DirectLineClientBase directLineClient, IFileHandler fileHandler)
+ {
+ this.logger = logger;
+ this.directLineClient = directLineClient;
+ this.fileHandler = fileHandler;
+ }
+
+ ///
+ /// Run a chat transcript test from a directory or file
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// boolean depending if the execution was successful or not
+ public async Task RunTranscriptTestAsync(DirectLineOptions options, string path, bool verbose = false, CancellationToken cancellationToken = default)
+ {
+ logger.Information("The test has started...");
+
+ try
+ {
+ var totalTests = 0;
+ var failedTests = 0;
+ var fileAttributes = fileHandler.GetFileAttributes(path);
+ var fileList = new List();
+
+ if (fileAttributes.HasFlag(FileAttributes.Directory))
+ {
+ // The path is a directory
+ fileList = fileHandler.GetFilesFromDirectory(path, "*.json").ToList();
+ }
+ else
+ {
+ // The path is just a file
+ fileList.Add(path);
+ }
+
+ logger.Information($"Using Direct Line endpoint: {options.RegionalEndpoint}");
+
+ foreach (var file in fileList)
+ {
+ logger.Information($"Testing with file: {file}");
+
+ var activityList = await GetActivitiesFromTranscriptFile(file);
+
+ if (activityList.list_of_conversations.Count > 0)
+ {
+ // The file has multiple conversations
+ foreach (var actlist in activityList.list_of_conversations)
+ {
+ totalTests++;
+ var result = await ExecuteTranscriptAsync(options, actlist, file, verbose, cancellationToken).ConfigureAwait(false);
+ if (!result) failedTests++;
+ }
+ }
+ else
+ {
+ // The file has a single conversation
+ totalTests++;
+ var result = await ExecuteTranscriptAsync(options, activityList, file, verbose, cancellationToken).ConfigureAwait(false);
+ if (!result) failedTests++;
+ }
+ }
+
+ logger.Information($"Test results: {totalTests - failedTests} passed, {totalTests} total");
+
+ return failedTests == 0;
+ }
+ catch (JsonReaderException ex)
+ {
+ logger.ForegroundColor($"An error occurred while running the test. Details: The json file used as an input is not valid.", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ catch (Exception ex)
+ {
+ logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ }
+
+ ///
+ /// Run a scale test, sending N conversations to the bot
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// boolean depending if the execution was successful or not
+ public async Task RunScaleTestAsync(DirectLineOptions options, int totalAttempts, string path, bool verbose)
+ {
+ logger.Information($"Running a Scale test (sending chat transcript {totalAttempts} times).");
+
+ try
+ {
+ var succeededAttempts = 0;
+ for (int i = 0; i < totalAttempts; i++)
+ {
+ if (verbose)
+ {
+ logger.Information($"Attempt {i+1}...");
+ }
+ try
+ {
+ var result = await RunTranscriptTestAsync(options, path, verbose);
+ if (result)
+ {
+ succeededAttempts++;
+ }
+ }
+ catch
+ {
+ logger.ForegroundColor($"The chat transcript failed.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ }
+ }
+
+ if (succeededAttempts == totalAttempts)
+ {
+ logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
+ logger.ForegroundColor($"Scale test ended succesfully.", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
+ return true;
+ }
+ else
+ {
+ logger.ForegroundColor($"The chat transcript was sent to the bot successfully in {succeededAttempts} attempts from {totalAttempts}.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Scale test ended with errors.", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ return false;
+ }
+ }
+ catch (Exception ex)
+ {
+ logger.ForegroundColor($"An error occurred while running the test. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ }
+
+ ///
+ /// Convert a .chat file into a .json file
+ ///
+ ///
+ ///
+ /// boolean depending if the conversion was successful or not
+ public bool ConvertChatFileToJSON(string path, string outputFile)
+ {
+ try
+ {
+ logger.Information($"The conversion process has started...");
+
+ var line = string.Empty;
+ var activityListContainer = new List>();
+ var activityList = new List();
+ string filepath = fileHandler.GetFullPath(outputFile);
+
+ object? channelData = null;
+
+ if (!filepath.EndsWith(".json"))
+ {
+ throw new ArgumentException("The output file should have a .json extension.");
+ }
+ if (!path.EndsWith(".chat"))
+ {
+ throw new ArgumentException("The input file should have a .chat extension.");
+ }
+
+ using (StreamReader sr = new StreamReader(path))
+ {
+ fileHandler.DeleteFile(filepath);
+
+ while (null != (line = sr.ReadLine()))
+ {
+ var activity = new Models.Activities.Activity();
+ if (string.IsNullOrWhiteSpace(line.Trim()))
+ {
+ continue;
+ }
+ else if (line.Trim().Equals("", StringComparison.InvariantCultureIgnoreCase))
+ {
+ activityListContainer.Add(activityList);
+ activityList = new List();
+ //reset channeldata for each conversation
+ channelData = null;
+ continue;
+ }
+ else if (line.StartsWith("user:"))
+ {
+ var userReg = new Regex(Regex.Escape("user:"));
+ var userText = userReg.Replace(line, string.Empty, 1).Trim();
+ if (string.IsNullOrEmpty(userText))
+ {
+ throw new ArgumentException("The user message is null or empty.");
+ }
+ else
+ {
+ activity = new Models.Activities.Activity
+ {
+ Type = Helpers.ActivityTypes.Message,
+ Text = userText,
+ From = new From(string.Empty, 1),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
+ ChannelData = channelData
+ };
+ }
+ }
+ else if (line.StartsWith("userEvent:"))
+ {
+ var userEventRegex = new Regex(Regex.Escape("userEvent:"));
+ var userEventText = userEventRegex.Replace(line, string.Empty, 1).Trim();
+
+ var userEventInfo = JObject.Parse(userEventText);
+
+ if (string.IsNullOrEmpty(userEventText) || userEventInfo == null)
+ {
+ throw new ArgumentException("The userEvent is invalid");
+ }
+ else
+ {
+ activity = new Models.Activities.Activity
+ {
+ Type = ActivityTypes.Event,
+ From = new From("user", 1),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
+ ChannelData = channelData,
+ Name = (string)userEventInfo["Name"],
+ Value = (string)userEventInfo["Value"]
+ };
+ }
+ }
+ else if (line.StartsWith("userMessage:"))
+ {
+ var userMessageRegex = new Regex(Regex.Escape("userMessage:"));
+ var userMessageText = userMessageRegex.Replace(line, string.Empty, 1).Trim();
+
+ var userMessageInfo = JObject.Parse(userMessageText);
+
+ if (string.IsNullOrEmpty(userMessageText) || userMessageInfo == null)
+ {
+ throw new ArgumentException("The userMessage is invalid");
+ }
+ else
+ {
+ activity = new Models.Activities.Activity
+ {
+ Type = ActivityTypes.Message,
+ From = new From("user", 1),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
+ ChannelData = channelData,
+ Text = (string)userMessageInfo["Text"],
+ Value = (string)userMessageInfo["Value"]
+ };
+ }
+ }
+ else if (line.StartsWith("bot:"))
+ {
+ var botReg = new Regex(Regex.Escape("bot:"));
+ var botText = botReg.Replace(line, string.Empty, 1).Trim();
+ if (string.IsNullOrEmpty(botText))
+ {
+ throw new ArgumentException("The bot message is null or empty.");
+ }
+ else
+ {
+ activity = new Models.Activities.Activity
+ {
+ Type = Helpers.ActivityTypes.Message,
+ Text = botText,
+ From = new From(string.Empty, 0),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow)
+ };
+ }
+ }
+ else if (line.StartsWith("botEvent:"))
+ {
+ var botEventRegex = new Regex(Regex.Escape("botEvent:"));
+ var botEventText = botEventRegex.Replace(line, string.Empty, 1).Trim();
+
+ var botEventInfo = JObject.Parse(botEventText);
+
+ if (string.IsNullOrEmpty(botEventText) || botEventInfo == null)
+ {
+ throw new ArgumentException("The botEvent is invalid");
+ }
+ else
+ {
+ activity = new Models.Activities.Activity
+ {
+ Type = ActivityTypes.Event,
+ From = new From("bot", 0),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
+ ChannelData = channelData,
+ Name = (string)botEventInfo["Name"],
+ Value = (string)botEventInfo["Value"]
+ };
+ }
+ }
+ else if (line.StartsWith("suggested:"))
+ {
+ var suggestionRegex = new Regex(Regex.Escape("suggested:"));
+ var suggestionText = suggestionRegex.Replace(line, string.Empty, 1).Trim();
+
+ if (string.IsNullOrEmpty(suggestionText))
+ {
+ throw new ArgumentException("The suggested message is null or empty.");
+ }
+
+ var suggestionList = suggestionText.Split('|', StringSplitOptions.RemoveEmptyEntries).ToList();
+ var intentCandidates = new List();
+
+ foreach (var suggestion in suggestionList)
+ {
+ intentCandidates.Add(new IntentCandidate
+ {
+ IntentScore = new IntentScore()
+ {
+ Title = suggestion
+ }
+ });
+ }
+ activity = new Models.Activities.Activity
+ {
+ ValueType = "IntentCandidates",
+ Type = Helpers.ActivityTypes.Trace,
+ From = new From(string.Empty, 0),
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
+ Value = new Value()
+ {
+ IntentCandidates = intentCandidates
+ }
+ };
+ }
+ else if (line.StartsWith("channelData:"))
+ {
+ var channelDataRegex = new Regex("channelData:");
+ var channelDataText = channelDataRegex.Replace(line, string.Empty, 1).Trim();
+
+ if (string.IsNullOrEmpty(channelDataText))
+ {
+ channelData = null;
+ continue;
+ }
+ else
+ {
+ channelData = JObject.Parse(channelDataText);
+
+ // if this is the first thing in the conversation (nothing in the activity list yet), add a startConversation event
+ if(activityList.Count == 0)
+ {
+ activity = new Models.Activities.Activity
+ {
+ Type = Helpers.ActivityTypes.Event,
+ From = new From(string.Empty, 1),
+ ChannelData = channelData,
+ Timestamp = ToUnixTimeSeconds(DateTime.UtcNow),
+ Name = "StartConversation"
+ };
+ }
+ else
+ {
+ continue;
+ }
+ }
+ }
+ else
+ {
+ throw new Exception("The input file format is not valid.");
+ }
+
+ activityList.Add(activity);
+ }
+ activityListContainer.Add(activityList);
+
+ var activities = new List();
+ foreach (var list in activityListContainer)
+ {
+ activities.Add($"{{\"Activities\":{string.Join(',', JsonConvert.SerializeObject(list))}}}");
+ }
+ fileHandler.WriteToFile(filepath, $"{{\"list_of_conversations\":[{string.Join(",", activities)}]}}");
+ }
+
+ logger.Information($"The conversion process ended.");
+ return true;
+ }
+ catch (Exception ex)
+ {
+ logger.ForegroundColor($"An error occurred while converting the .chat transcript. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ }
+
+ ///
+ /// Execute the conversation on the transcript
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// boolean depending if the conversion was successful or not
+ ///
+ private async Task ExecuteTranscriptAsync(DirectLineOptions options, ActivityList activityList, string path, bool verbose = false, CancellationToken cancellationToken = default)
+ {
+ var logRecords = new List();
+ var activities = new List();
+ var timer = new Stopwatch();
+ var testFailed = false;
+ var userUtterance = string.Empty;
+
+ timer.Start();
+ path = fileHandler.GetFullPath(path);
+
+ try
+ {
+ using (directLineClient)
+ {
+ foreach (var activity in activityList.Activities)
+ {
+ switch (activity.From.Role)
+ {
+ case RoleTypes.User:
+ var sendActivity = new Activity
+ {
+ Type = activity.Type,
+ Text = activity.Text,
+ ChannelData = activity.ChannelData,
+ Name= activity.Name,
+ Value = activity.Value
+ };
+
+ userUtterance = sendActivity.Text;
+ if (verbose)
+ {
+ logger.Information($"User sends: {sendActivity.Text}");
+ }
+
+ await directLineClient.SendActivityAsync(sendActivity, cancellationToken).ConfigureAwait(false);
+ break;
+
+ case RoleTypes.Bot:
+ if (IgnoreActivity(activity))
+ {
+ break;
+ }
+
+ var receivedActivity = new Activity();
+ var receivedOptions = new List();
+ var expectedOptions = new List();
+
+ if (activities.Count == 0)
+ {
+ activities = await directLineClient.ReceiveActivitiesAsync(cancellationToken).ConfigureAwait(false);
+
+ // Get the first activity from the bot response
+ receivedActivity = activities.FirstOrDefault();
+ activities.Remove(receivedActivity);
+
+ if (verbose)
+ {
+ logger.Information($"Bot sends: {receivedActivity.Text}");
+ }
+ if (receivedActivity.SuggestedActions != null)
+ {
+ // Get the suggested topics from the activity if any
+ receivedOptions = receivedActivity.SuggestedActions?.Actions?.Where(o => !o.Title.Equals(BotDefaultMessages.NoneOfThese, StringComparison.InvariantCultureIgnoreCase)).Select(a => a.Title).ToList();
+
+ if (verbose)
+ {
+ logger.Information($"\t{string.Join(" | ", receivedOptions)}");
+ }
+ }
+ }
+ else
+ {
+ // Get the first activity from the list
+ receivedActivity = activities.FirstOrDefault();
+ activities.Remove(receivedActivity);
+
+ if (verbose)
+ {
+ logger.Information($"Bot sends: {receivedActivity.Text}");
+ }
+ }
+
+ var csvRecord = new LogCSV()
+ {
+ BotId = receivedActivity.From.Id,
+ ConversationId = receivedActivity.Conversation.Id,
+ SessionDate = DateTime.Now.ToString(),
+ UserUtterance = userUtterance,
+ ExpectedResponse = activity.Text,
+ ReceivedResponse = receivedActivity.Text,
+ TestFile = path
+ };
+
+ if (receivedActivity.Text != null && receivedActivity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase))
+ {
+ // activity.Value redefined as type object?, so cast it back to type Value for this particular situation
+ expectedOptions = ((Value)(activity.Value))?.IntentCandidates != null ? ((Value)(activity.Value))?.IntentCandidates?.Select(o => o.IntentScore.Title).ToList() : new List() { "No suggested topics found" };
+
+ for (int i = 0; i < receivedOptions.Count; i++)
+ {
+ if (i == 0) csvRecord.DYM_Option1 = receivedOptions[i];
+ if (i == 1) csvRecord.DYM_Option2 = receivedOptions[i];
+ if (i == 2) csvRecord.DYM_Option3 = receivedOptions[i];
+ }
+
+ if (expectedOptions.Count != receivedOptions.Count)
+ {
+ testFailed = true;
+ }
+ else
+ {
+ foreach (var suggestion in receivedOptions)
+ {
+ if (!expectedOptions.Any(option => option.Equals(suggestion, StringComparison.InvariantCultureIgnoreCase)))
+ {
+ testFailed = true;
+ break;
+ }
+ }
+ }
+
+ if (testFailed)
+ {
+ logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Expected:\t{string.Join(" | ", expectedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Received:\t{string.Join(" | ", receivedOptions)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ }
+ }
+ else
+ {
+ if (!AssertActivity(activity, receivedActivity))
+ {
+ logger.ForegroundColor($"Test script failed", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+
+ if (!string.IsNullOrEmpty(activity.Text))
+ {
+ logger.ForegroundColor($"Expected: {activity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Received: {receivedActivity.Text}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Line number: {activity.LineNumber}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ }
+
+ if (activity.Attachments != null && activity.Attachments.Count > 0)
+ {
+ var expectedAttachments = activity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
+ var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
+
+ logger.ForegroundColor($"Expected: {string.Join(Environment.NewLine, expectedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ logger.ForegroundColor($"Received: {string.Join(Environment.NewLine, receivedAttachments)}", LoggerExtensions.LogLevel.Error, LoggerExtensions.Yellow);
+ }
+
+ testFailed = true;
+ }
+ }
+
+ userUtterance = string.Empty;
+ csvRecord.Result = testFailed ? "Failed" : "Passed";
+ logRecords.Add(csvRecord);
+ break;
+
+ default:
+ throw new InvalidOperationException($"Invalid script role {activity.From.Role}.");
+ }
+
+ if (testFailed) break;
+ }
+ }
+
+ timer.Stop();
+
+ if (!testFailed)
+ {
+ logger.ForegroundColor($"Test script passed", LoggerExtensions.LogLevel.Information, LoggerExtensions.Green);
+ }
+
+ logger.Information($"Time: {timer.Elapsed.TotalSeconds.ToString("0.00")} seconds");
+
+ return !testFailed;
+ }
+ catch (Exception ex)
+ {
+ logger.ForegroundColor($"An error occurred while validating the chat transcript file. Details: {ex.Message}", LoggerExtensions.LogLevel.Fatal, LoggerExtensions.Red);
+ return false;
+ }
+ finally
+ {
+ WriteCSVLog(logRecords);
+ }
+ }
+
+ ///
+ /// Receives an activityList from a transcript file
+ ///
+ ///
+ /// an activityList
+ private async Task GetActivitiesFromTranscriptFile(string fileName)
+ {
+ using var reader = new StreamReader(fileName);
+ var transcript = await reader.ReadToEndAsync().ConfigureAwait(false);
+ var activityList = JsonConvert.DeserializeObject(transcript);
+
+ if (activityList.Activities.Count == 0 && activityList.list_of_conversations.Count == 0)
+ {
+ throw new JsonReaderException();
+ }
+ else if (activityList.list_of_conversations.Count > 0)
+ {
+ foreach (var activities in activityList.list_of_conversations)
+ {
+ AddTextLineNumber(activities, fileName);
+ }
+ }
+ else
+ {
+ AddTextLineNumber(activityList, fileName);
+ }
+
+ return activityList;
+ }
+
+ ///
+ /// Gets the line number for the activity text
+ ///
+ ///
+ ///
+ private void AddTextLineNumber(ActivityList activityList, string fileName)
+ {
+ var lineNumber = 0;
+ var lines = File.ReadLines(fileName).ToList();
+
+ foreach (var activity in activityList.Activities.Where(a => a.IsMessageActivityWithText()))
+ {
+ // Added line number for Text messages
+ lineNumber = lines.FindIndex(lineNumber, line => line.Contains(activity.Text)) + 1;
+ activity.LineNumber = lineNumber;
+ }
+ }
+
+ ///
+ /// Write CSV log
+ ///
+ ///
+ private void WriteCSVLog(List records)
+ {
+ var csvLogPath = Path.Combine(Environment.CurrentDirectory, "logFile.csv");
+ var config = new CsvConfiguration(CultureInfo.InvariantCulture)
+ {
+ // Don't write the header again.
+ HasHeaderRecord = false,
+ // Use the system list separator
+ Delimiter = CultureInfo.CurrentCulture.TextInfo.ListSeparator
+ };
+
+ if (!File.Exists(csvLogPath))
+ {
+ using (var streamWriter = new StreamWriter(csvLogPath))
+ {
+ using (var csvWriter = new CsvWriter(streamWriter, config))
+ {
+ csvWriter.WriteHeader();
+ csvWriter.NextRecord();
+ }
+ }
+ }
+
+ using (var stream = File.Open(csvLogPath, FileMode.Append))
+ {
+ using (var streamWriter = new StreamWriter(stream))
+ {
+ using (var csvWriter = new CsvWriter(streamWriter, config))
+ {
+ csvWriter.WriteRecords(records);
+ }
+ }
+ }
+ }
+
+ ///
+ /// Check if adaptive cards structure are equal
+ ///
+ ///
+ ///
+ /// boolean
+ private bool AssertActivity(Models.Activities.Activity expectedActivity, Activity receivedActivity)
+ {
+ bool result = true;
+ if (!string.IsNullOrEmpty(expectedActivity.Text) && !string.IsNullOrEmpty(receivedActivity.Text))
+ {
+ // Replace unwanted characters
+ receivedActivity.Text = receivedActivity.Text.Replace((char)0xA0, ' ');
+
+ string pattern = ExtractRegex(expectedActivity.Text);
+ if (!string.IsNullOrEmpty(pattern))
+ {
+ // If the line contains a regex pattern it will check if matches
+ return Regex.IsMatch(receivedActivity.Text, pattern);
+ }
+ // Allow for new lines in chat file, json and/or activity text by completely unescaping both strings - Issue 218
+ else if (!expectedActivity.Text.CompletelyUnescape().Equals(receivedActivity.Text.CompletelyUnescape(), StringComparison.InvariantCultureIgnoreCase))
+ {
+ // This is a simple text to compare
+ return false;
+ }
+ }
+ else if (expectedActivity.Attachments != null && expectedActivity.Attachments.Count == receivedActivity.Attachments.Count)
+ {
+ // This is an adaptive card, so the structure comparison will be executed
+ var expectedAttachments = expectedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
+ var receivedAttachments = receivedActivity.Attachments.Select(a => JsonConvert.SerializeObject(a.Content)).ToList();
+ var settings = new AdaptiveCardTranslatorSettings();
+
+ for (int i = 0; i < expectedActivity.Attachments.Count; i++)
+ {
+ var expectedCard = AdaptiveCard.GetCardWithoutValues(expectedAttachments[i].ToJObject(true), settings);
+ var receivedCard = AdaptiveCard.GetCardWithoutValues(receivedAttachments[i].ToJObject(true), settings);
+ if (!expectedCard.Equals(receivedCard, StringComparison.InvariantCultureIgnoreCase))
+ {
+ return false;
+ }
+ }
+ }
+ else
+ {
+ return false;
+ }
+
+ return result;
+ }
+
+ ///
+ /// validate if the activity should be ignored
+ ///
+ ///
+ /// boolean
+ private bool IgnoreActivity(Models.Activities.Activity activity)
+ {
+ // Ignore trace activities unless it is an IntentCandidates type one. Also, ignore the DYM message as it is sent in the previous activity
+ return (activity.Type == Helpers.ActivityTypes.Trace
+ && !activity.ValueType.Equals("IntentCandidates", StringComparison.InvariantCultureIgnoreCase))
+ || (activity.Type == Helpers.ActivityTypes.Message
+ && activity.Text != null
+ && activity.Text.Equals(BotDefaultMessages.DYM, StringComparison.InvariantCultureIgnoreCase));
+ }
+
+ ///
+ /// Extract the regex pattern
+ ///
+ ///
+ /// path
+ private string ExtractRegex(string input)
+ {
+ if (input == null)
+ {
+ return null;
+ }
+ string regexPattern = "<\\((.*?)\\)>"; // The regex pattern to search for
+ Regex regex = new Regex(regexPattern);
+ Match match = regex.Match(input);
+ if (match.Success)
+ {
+ return match.Groups[1].Value; // Extract the REGEX
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+ ///
+ /// Convert a datetime to Unix time format
+ ///
+ ///
+ /// path
+ private int ToUnixTimeSeconds(DateTime date)
+ {
+ DateTime point = new DateTime(1970, 1, 1);
+ TimeSpan time = date.Subtract(point);
+
+ return (int)time.TotalSeconds;
+ }
+ }
+}
diff --git a/PVATestFramework/PVATestFramework/Models/Activities/Activity.cs b/PVATestFramework/PVATestFramework/Models/Activities/Activity.cs
index bb5f0837..be148427 100644
--- a/PVATestFramework/PVATestFramework/Models/Activities/Activity.cs
+++ b/PVATestFramework/PVATestFramework/Models/Activities/Activity.cs
@@ -49,11 +49,16 @@ public static bool IsMessageActivityWithText(this Activity activity)
public class Attachment
{
+ public Attachment(string contentType, Object content)
+ {
+ ContentType = contentType;
+ Content = content;
+ }
[JsonProperty("contentType")]
public string ContentType { get; set; }
[JsonProperty("content")]
- public Content Content { get; set; }
+ public Object Content { get; set; }
}
public class Action
@@ -121,6 +126,17 @@ public class Content
[JsonProperty("version")]
public string Version { get; set; }
}
+ public class BasicCardContent
+ {
+ [JsonProperty("title")]
+ public string Title { get; set; }
+
+ [JsonProperty("images")]
+ public List Images { get; set; }
+
+ [JsonProperty("buttons")]
+ public List Buttons { get; set; }
+ }
public class Data
{
diff --git a/PVATestFramework/PVATestFramework/PVATestFramework.sln b/PVATestFramework/PVATestFramework/PVATestFramework.sln
new file mode 100644
index 00000000..c71b0b68
--- /dev/null
+++ b/PVATestFramework/PVATestFramework/PVATestFramework.sln
@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.002.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PVATestFramework", "PVATestFramework.csproj", "{93399C26-6CC4-4C53-987B-166122B78300}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Release|Any CPU = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {93399C26-6CC4-4C53-987B-166122B78300}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {93399C26-6CC4-4C53-987B-166122B78300}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {93399C26-6CC4-4C53-987B-166122B78300}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {93399C26-6CC4-4C53-987B-166122B78300}.Release|Any CPU.Build.0 = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+ GlobalSection(ExtensibilityGlobals) = postSolution
+ SolutionGuid = {C68EE9E9-3236-418C-8F2F-23E9420EA006}
+ EndGlobalSection
+EndGlobal