Remove the abstraction for token counting from the main evaluation API (#6320)

This change is being made because there is still some uncertainty around what a general-purpose token counting abstraction (that supports all kinds of future models, and all kinds of input modalities) should look like at the moment. We do not want to bake in an API that only supports text-based inputs for the models and use cases that are prevalent today, since it would be a potential breaking change to change this API after we release a stable version of the evaluation APIs.

We can always reintroduce the token counting support in a non-breaking fashion in the future if and when there is more clarity on what a general-purpose token counting abstraction should look like, or if and when a general-purpose token counting abstraction is introduced in a lower layer (Microsoft.Extensions.AI) in the future.

In the meanwhile, callers should still be able to use the `Microsoft.ML.Tokenizers` library directly to count tokens in text-based content and trim down the conversation history before calling `EvaluateAsync()` if needed.

Fixes #6234

Author: shyamnamboodiripad

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/ChatConversationEvaluator.cs

@@ -55,100 +55,8 @@ public virtual async ValueTask<EvaluationResult> EvaluateAsync(
             return result;
         }
 
-        (ChatMessage? userRequest, List<ChatMessage> history) = GetUserRequestAndHistory(messages);
-
-        int inputTokenLimit = 0;
-        int ignoredMessagesCount = 0;
-
-        if (chatConfiguration.TokenCounter is not null)
-        {
-            IEvaluationTokenCounter tokenCounter = chatConfiguration.TokenCounter;
-            inputTokenLimit = tokenCounter.InputTokenLimit;
-            int tokenBudget = inputTokenLimit;
-
-            void OnTokenBudgetExceeded()
-            {
-                EvaluationDiagnostic tokenBudgetExceeded =
-                    EvaluationDiagnostic.Error(
-                        $"Evaluation failed because the specified limit of {inputTokenLimit} input tokens was exceeded.");
-
-                result.AddDiagnosticsToAllMetrics(tokenBudgetExceeded);
-            }
-
-            if (!string.IsNullOrWhiteSpace(SystemPrompt))
-            {
-                tokenBudget -= tokenCounter.CountTokens(SystemPrompt!);
-                if (tokenBudget < 0)
-                {
-                    OnTokenBudgetExceeded();
-                    return result;
-                }
-            }
-
-            string baseEvaluationPrompt =
-                await RenderEvaluationPromptAsync(
-                    userRequest,
-                    modelResponse,
-                    includedHistory: [],
-                    additionalContext,
-                    cancellationToken).ConfigureAwait(false);
-
-            tokenBudget -= tokenCounter.CountTokens(baseEvaluationPrompt);
-            if (tokenBudget < 0)
-            {
-                OnTokenBudgetExceeded();
-                return result;
-            }
-
-            if (history.Count > 0 && !IgnoresHistory)
-            {
-                if (history.Count == 1)
-                {
-                    (bool canRender, tokenBudget) =
-                        await CanRenderAsync(
-                            history[0],
-                            tokenBudget,
-                            chatConfiguration,
-                            cancellationToken).ConfigureAwait(false);
-
-                    if (!canRender)
-                    {
-                        ignoredMessagesCount = 1;
-                        history = [];
-                    }
-                }
-                else
-                {
-                    int totalMessagesCount = history.Count;
-                    int includedMessagesCount = 0;
-
-                    history.Reverse();
-
-                    foreach (ChatMessage message in history)
-                    {
-                        cancellationToken.ThrowIfCancellationRequested();
-
-                        (bool canRender, tokenBudget) =
-                            await CanRenderAsync(
-                                message,
-                                tokenBudget,
-                                chatConfiguration,
-                                cancellationToken).ConfigureAwait(false);
-
-                        if (!canRender)
-                        {
-                            ignoredMessagesCount = totalMessagesCount - includedMessagesCount;
-                            history.RemoveRange(index: includedMessagesCount, count: ignoredMessagesCount);
-                            break;
-                        }
-
-                        includedMessagesCount++;
-                    }
-
-                    history.Reverse();
-                }
-            }
-        }
+        (ChatMessage? userRequest, List<ChatMessage> conversationHistory) =
+            GetUserRequestAndConversationHistory(messages);
 
         var evaluationMessages = new List<ChatMessage>();
         if (!string.IsNullOrWhiteSpace(SystemPrompt))
@@ -160,7 +68,7 @@ await CanRenderAsync(
             await RenderEvaluationPromptAsync(
                 userRequest,
                 modelResponse,
-                includedHistory: history,
+                conversationHistory,
                 additionalContext,
                 cancellationToken).ConfigureAwait(false);
 
@@ -172,84 +80,9 @@ await PerformEvaluationAsync(
             result,
             cancellationToken).ConfigureAwait(false);
 
-        if (inputTokenLimit > 0 && ignoredMessagesCount > 0)
-        {
-#pragma warning disable S103 // Lines should not be too long
-            result.AddDiagnosticsToAllMetrics(
-                EvaluationDiagnostic.Warning(
-                    $"The evaluation may be inconclusive because the oldest {ignoredMessagesCount} messages in the supplied conversation history were ignored in order to stay under the specified limit of {inputTokenLimit} input tokens."));
-#pragma warning restore S103
-        }
-
         return result;
     }
 
-    /// <summary>
-    /// Determines if there is sufficient <paramref name="tokenBudget"/> remaining to render the
-    /// supplied <paramref name="message"/> as part of the evaluation prompt that this <see cref="IEvaluator"/> uses.
-    /// </summary>
-    /// <param name="message">
-    /// A message that is part of the conversation history for the response being evaluated and that is to be rendered
-    /// as part of the evaluation prompt.
-    /// </param>
-    /// <param name="tokenBudget">
-    /// The number of tokens available for the rendering additional content as part of the evaluation prompt.
-    /// </param>
-    /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that this <see cref="IEvaluator"/> uses to perform the evaluation.
-    /// </param>
-    /// <param name="cancellationToken">A <see cref="CancellationToken"/> that can cancel the operation.</param>
-    /// <returns>
-    /// A tuple containing a <see langword="bool"/> indicating whether there is sufficient
-    /// <paramref name="tokenBudget"/> remaining to render the supplied <paramref name="message"/> as part of the
-    /// evaluation prompt, and an <see langword="int"/> containing the remaining token budget that would be available
-    /// once this <paramref name="message"/> is rendered.
-    /// </returns>
-    protected virtual ValueTask<(bool canRender, int remainingTokenBudget)> CanRenderAsync(
-        ChatMessage message,
-        int tokenBudget,
-        ChatConfiguration chatConfiguration,
-        CancellationToken cancellationToken)
-    {
-        _ = Throw.IfNull(message);
-        _ = Throw.IfNull(chatConfiguration);
-
-        IEvaluationTokenCounter? tokenCounter = chatConfiguration.TokenCounter;
-        if (tokenCounter is null)
-        {
-            return new ValueTask<(bool, int)>((true, tokenBudget));
-        }
-
-        string? author = message.AuthorName;
-        string role = message.Role.Value;
-        string content = message.Text ?? string.Empty;
-
-        int tokenCount =
-            string.IsNullOrWhiteSpace(author)
-                ? tokenCounter.CountTokens("[") +
-                    tokenCounter.CountTokens(role) +
-                    tokenCounter.CountTokens("] ") +
-                    tokenCounter.CountTokens(content) +
-                    tokenCounter.CountTokens("\n")
-                : tokenCounter.CountTokens("[") +
-                    tokenCounter.CountTokens(author!) +
-                    tokenCounter.CountTokens(" (") +
-                    tokenCounter.CountTokens(role) +
-                    tokenCounter.CountTokens(")] ") +
-                    tokenCounter.CountTokens(content) +
-                    tokenCounter.CountTokens("\n");
-
-        if (tokenCount > tokenBudget)
-        {
-            return new ValueTask<(bool, int)>((false, tokenBudget));
-        }
-        else
-        {
-            return new ValueTask<(bool, int)>((true, tokenBudget - tokenCount));
-        }
-    }
-
     /// <summary>
     /// Renders the supplied <paramref name="response"/> to a string that can be included as part of the evaluation
     /// prompt that this <see cref="IEvaluator"/> uses.
@@ -313,21 +146,21 @@ protected virtual ValueTask<string> RenderAsync(ChatMessage message, Cancellatio
     /// The request that produced the <paramref name="modelResponse"/> that is to be evaluated.
     /// </param>
     /// <param name="modelResponse">The response that is to be evaluated.</param>
-    /// <param name="includedHistory">
+    /// <param name="conversationHistory">
     /// The conversation history (excluding the <paramref name="userRequest"/> and <paramref name="modelResponse"/>)
     /// that is to be included as part of the evaluation prompt.
     /// </param>
     /// <param name="additionalContext">
     /// Additional contextual information (beyond that which is available in the <paramref name="userRequest"/> and
-    /// <paramref name="includedHistory"/>) that this <see cref="IEvaluator"/> may need to accurately evaluate the
+    /// <paramref name="conversationHistory"/>) that this <see cref="IEvaluator"/> may need to accurately evaluate the
     /// supplied <paramref name="modelResponse"/>.
     /// </param>
     /// <param name="cancellationToken">A <see cref="CancellationToken"/> that can cancel the operation.</param>
     /// <returns>The evaluation prompt.</returns>
     protected abstract ValueTask<string> RenderEvaluationPromptAsync(
         ChatMessage? userRequest,
         ChatResponse modelResponse,
-        IEnumerable<ChatMessage>? includedHistory,
+        IEnumerable<ChatMessage>? conversationHistory,
         IEnumerable<EvaluationContext>? additionalContext,
         CancellationToken cancellationToken);
 
@@ -351,8 +184,8 @@ protected abstract ValueTask<string> RenderEvaluationPromptAsync(
     /// <see cref="EvaluationMetric"/>s in the supplied <paramref name="result"/>.
     /// </summary>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that this <see cref="IEvaluator"/> uses to perform the evaluation.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that should be used if one or
+    /// more composed <see cref="IEvaluator"/>s use an AI model to perform evaluation.
     /// </param>
     /// <param name="evaluationMessages">
     /// The set of messages that are to be sent to the supplied <see cref="ChatConfiguration.ChatClient"/> to perform
@@ -370,11 +203,11 @@ protected abstract ValueTask PerformEvaluationAsync(
         EvaluationResult result,
         CancellationToken cancellationToken);
 
-    private (ChatMessage? userRequest, List<ChatMessage> history) GetUserRequestAndHistory(
+    private (ChatMessage? userRequest, List<ChatMessage> conversationHistory) GetUserRequestAndConversationHistory(
         IEnumerable<ChatMessage> messages)
     {
         ChatMessage? userRequest = null;
-        List<ChatMessage> history;
+        List<ChatMessage> conversationHistory;
 
         if (IgnoresHistory)
         {
@@ -383,22 +216,22 @@ protected abstract ValueTask PerformEvaluationAsync(
                     ? lastMessage
                     : null;
 
-            history = [];
+            conversationHistory = [];
         }
         else
         {
-            history = [.. messages];
-            int lastMessageIndex = history.Count - 1;
+            conversationHistory = [.. messages];
+            int lastMessageIndex = conversationHistory.Count - 1;
 
             if (lastMessageIndex >= 0 &&
-                history[lastMessageIndex] is ChatMessage lastMessage &&
+                conversationHistory[lastMessageIndex] is ChatMessage lastMessage &&
                 lastMessage.Role == ChatRole.User)
             {
                 userRequest = lastMessage;
-                history.RemoveAt(lastMessageIndex);
+                conversationHistory.RemoveAt(lastMessageIndex);
             }
         }
 
-        return (userRequest, history);
+        return (userRequest, conversationHistory);
     }
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/CoherenceEvaluator.cs

@@ -50,7 +50,7 @@ public sealed class CoherenceEvaluator : SingleNumericMetricEvaluator
     protected override async ValueTask<string> RenderEvaluationPromptAsync(
         ChatMessage? userRequest,
         ChatResponse modelResponse,
-        IEnumerable<ChatMessage>? includedHistory,
+        IEnumerable<ChatMessage>? conversationHistory,
         IEnumerable<EvaluationContext>? additionalContext,
         CancellationToken cancellationToken)
     {

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/EquivalenceEvaluator.cs

@@ -75,7 +75,7 @@ await base.EvaluateAsync(
     protected override async ValueTask<string> RenderEvaluationPromptAsync(
         ChatMessage? userRequest,
         ChatResponse modelResponse,
-        IEnumerable<ChatMessage>? includedHistory,
+        IEnumerable<ChatMessage>? conversationHistory,
         IEnumerable<EvaluationContext>? additionalContext,
         CancellationToken cancellationToken)
     {

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/FluencyEvaluator.cs

@@ -49,7 +49,7 @@ public sealed class FluencyEvaluator : SingleNumericMetricEvaluator
     protected override async ValueTask<string> RenderEvaluationPromptAsync(
         ChatMessage? userRequest,
         ChatResponse modelResponse,
-        IEnumerable<ChatMessage>? includedHistory,
+        IEnumerable<ChatMessage>? conversationHistory,
         IEnumerable<EvaluationContext>? additionalContext,
         CancellationToken cancellationToken)
     {

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/GroundednessEvaluator.cs

@@ -77,7 +77,7 @@ await base.EvaluateAsync(
     protected override async ValueTask<string> RenderEvaluationPromptAsync(
         ChatMessage? userRequest,
         ChatResponse modelResponse,
-        IEnumerable<ChatMessage>? includedHistory,
+        IEnumerable<ChatMessage>? conversationHistory,
         IEnumerable<EvaluationContext>? additionalContext,
         CancellationToken cancellationToken)
     {
@@ -99,9 +99,9 @@ userRequest is not null
             _ = builder.AppendLine();
         }
 
-        if (includedHistory is not null)
+        if (conversationHistory is not null)
         {
-            foreach (ChatMessage message in includedHistory)
+            foreach (ChatMessage message in conversationHistory)
             {
                 _ = builder.Append(await RenderAsync(message, cancellationToken).ConfigureAwait(false));
             }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/RelevanceTruthAndCompletenessEvaluator.cs

@@ -93,7 +93,7 @@ protected override EvaluationResult InitializeResult()
     protected override async ValueTask<string> RenderEvaluationPromptAsync(
         ChatMessage? userRequest,
         ChatResponse modelResponse,
-        IEnumerable<ChatMessage>? includedHistory,
+        IEnumerable<ChatMessage>? conversationHistory,
         IEnumerable<EvaluationContext>? additionalContext,
         CancellationToken cancellationToken)
     {
@@ -107,9 +107,9 @@ userRequest is not null
                 : string.Empty;
 
         var builder = new StringBuilder();
-        if (includedHistory is not null)
+        if (conversationHistory is not null)
         {
-            foreach (ChatMessage message in includedHistory)
+            foreach (ChatMessage message in conversationHistory)
             {
                 _ = builder.Append(await RenderAsync(message, cancellationToken).ConfigureAwait(false));
             }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Utilities/JsonOutputFixer.cs

@@ -24,7 +24,7 @@ internal static ReadOnlySpan<char> TrimMarkdownDelimiters(string json)
         // Trim 'json' marker from markdown if it exists.
         const string JsonMarker = "json";
         int markerLength = JsonMarker.Length;
-        if (trimmed.Length > markerLength && trimmed[0..markerLength].SequenceEqual(JsonMarker.AsSpan()))
+        if (trimmed.Length > markerLength && trimmed.Slice(0, markerLength).SequenceEqual(JsonMarker.AsSpan()))
         {
             trimmed = trimmed.Slice(markerLength);
         }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs

@@ -29,10 +29,9 @@ public static class AzureStorageReportingConfiguration
     /// survive in the cache before they are considered expired and evicted.
     /// </param>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <paramref name="evaluators"/> included in the
-    /// returned <see cref="ReportingConfiguration"/>. Can be omitted if none of the included
-    /// <paramref name="evaluators"/> are AI-based.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by AI-based
+    /// <paramref name="evaluators"/> included in the returned <see cref="ReportingConfiguration"/>. Can be omitted if
+    /// none of the included <paramref name="evaluators"/> are AI-based.
     /// </param>
     /// <param name="enableResponseCaching">
     /// <see langword="true"/> to enable caching of AI responses; <see langword="false"/> otherwise.