| | | 1 | | using System.Collections.Generic; |
| | | 2 | | using System.Linq; |
| | | 3 | | using System.Text.Json; |
| | | 4 | | using System.Text.Json.Serialization; |
| | | 5 | | using EHonda.KicktippAi.Core; |
| | | 6 | | using Microsoft.Extensions.Logging; |
| | | 7 | | using OpenAI.Chat; |
| | | 8 | | |
| | | 9 | | namespace OpenAiIntegration; |
| | | 10 | | |
| | | 11 | | /// <summary> |
| | | 12 | | /// Service for predicting match outcomes using OpenAI models |
| | | 13 | | /// </summary> |
| | | 14 | | public class PredictionService : IPredictionService |
| | | 15 | | { |
| | | 16 | | private readonly ChatClient _chatClient; |
| | | 17 | | private readonly ILogger<PredictionService> _logger; |
| | | 18 | | private readonly ICostCalculationService _costCalculationService; |
| | | 19 | | private readonly ITokenUsageTracker _tokenUsageTracker; |
| | | 20 | | private readonly IInstructionsTemplateProvider _templateProvider; |
| | | 21 | | private readonly string _model; |
| | | 22 | | private readonly string _instructionsTemplate; |
| | | 23 | | private readonly string _instructionsTemplateWithJustification; |
| | | 24 | | private readonly string _bonusInstructionsTemplate; |
| | | 25 | | private readonly string _matchPromptPath; |
| | | 26 | | private readonly string _matchPromptPathWithJustification; |
| | | 27 | | private readonly string _bonusPromptPath; |
| | | 28 | | |
| | | 29 | | public PredictionService( |
| | | 30 | | ChatClient chatClient, |
| | | 31 | | ILogger<PredictionService> logger, |
| | | 32 | | ICostCalculationService costCalculationService, |
| | | 33 | | ITokenUsageTracker tokenUsageTracker, |
| | | 34 | | IInstructionsTemplateProvider templateProvider, |
| | | 35 | | string model) |
| | | 36 | | { |
| | | 37 | | _chatClient = chatClient ?? throw new ArgumentNullException(nameof(chatClient)); |
| | | 38 | | _logger = logger ?? throw new ArgumentNullException(nameof(logger)); |
| | | 39 | | _costCalculationService = costCalculationService ?? throw new ArgumentNullException(nameof(costCalculationServic |
| | | 40 | | _tokenUsageTracker = tokenUsageTracker ?? throw new ArgumentNullException(nameof(tokenUsageTracker)); |
| | | 41 | | _templateProvider = templateProvider ?? throw new ArgumentNullException(nameof(templateProvider)); |
| | | 42 | | _model = model ?? throw new ArgumentNullException(nameof(model)); |
| | | 43 | | |
| | | 44 | | var (matchTemplate, matchPath) = _templateProvider.LoadMatchTemplate(_model, includeJustification: false); |
| | | 45 | | var (matchJustificationTemplate, matchJustificationPath) = _templateProvider.LoadMatchTemplate(_model, includeJu |
| | | 46 | | var (bonusTemplate, bonusPath) = _templateProvider.LoadBonusTemplate(_model); |
| | | 47 | | |
| | | 48 | | _instructionsTemplate = matchTemplate; |
| | | 49 | | _instructionsTemplateWithJustification = matchJustificationTemplate; |
| | | 50 | | _bonusInstructionsTemplate = bonusTemplate; |
| | | 51 | | _matchPromptPath = matchPath; |
| | | 52 | | _matchPromptPathWithJustification = matchJustificationPath; |
| | | 53 | | _bonusPromptPath = bonusPath; |
| | | 54 | | } |
| | | 55 | | |
| | | 56 | | public async Task<Prediction?> PredictMatchAsync( |
| | | 57 | | Match match, |
| | | 58 | | IEnumerable<DocumentContext> contextDocuments, |
| | | 59 | | bool includeJustification = false, |
| | | 60 | | CancellationToken cancellationToken = default) |
| | | 61 | | { |
| | | 62 | | _logger.LogInformation("Generating prediction for match: {HomeTeam} vs {AwayTeam} at {StartTime}", |
| | | 63 | | match.HomeTeam, match.AwayTeam, match.StartsAt); |
| | | 64 | | |
| | | 65 | | try |
| | | 66 | | { |
| | | 67 | | // Build the instructions by combining template with context |
| | | 68 | | var instructions = BuildInstructions(contextDocuments, includeJustification); |
| | | 69 | | |
| | | 70 | | // Create match JSON |
| | | 71 | | var matchJson = CreateMatchJson(match); |
| | | 72 | | |
| | | 73 | | _logger.LogDebug("Instructions length: {InstructionsLength} characters", instructions.Length); |
| | | 74 | | _logger.LogDebug("Context documents: {ContextCount}", contextDocuments.Count()); |
| | | 75 | | _logger.LogDebug("Match JSON: {MatchJson}", matchJson); |
| | | 76 | | |
| | | 77 | | // Create messages for the chat completion |
| | | 78 | | var messages = new List<ChatMessage> |
| | | 79 | | { |
| | | 80 | | new SystemChatMessage(instructions), |
| | | 81 | | new UserChatMessage(matchJson) |
| | | 82 | | }; |
| | | 83 | | |
| | | 84 | | _logger.LogDebug("Calling OpenAI API for prediction"); |
| | | 85 | | |
| | | 86 | | // Call OpenAI with structured output format |
| | | 87 | | var response = await _chatClient.CompleteChatAsync( |
| | | 88 | | messages, |
| | | 89 | | new ChatCompletionOptions |
| | | 90 | | { |
| | | 91 | | MaxOutputTokenCount = 10_000, // Safeguard against high costs |
| | | 92 | | ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat( |
| | | 93 | | jsonSchemaFormatName: "match_prediction", |
| | | 94 | | jsonSchema: BinaryData.FromBytes(BuildPredictionJsonSchema(includeJustification)), |
| | | 95 | | jsonSchemaIsStrict: true) |
| | | 96 | | }, |
| | | 97 | | cancellationToken); |
| | | 98 | | |
| | | 99 | | // Parse the structured response |
| | | 100 | | var predictionJson = response.Value.Content[0].Text; |
| | | 101 | | _logger.LogDebug("Received prediction JSON: {PredictionJson}", predictionJson); |
| | | 102 | | |
| | | 103 | | var prediction = ParsePrediction(predictionJson); |
| | | 104 | | |
| | | 105 | | _logger.LogInformation("Prediction generated: {HomeGoals}-{AwayGoals} for {HomeTeam} vs {AwayTeam}", |
| | | 106 | | prediction.HomeGoals, prediction.AwayGoals, match.HomeTeam, match.AwayTeam); |
| | | 107 | | |
| | | 108 | | // Log token usage and cost breakdown |
| | | 109 | | var usage = response.Value.Usage; |
| | | 110 | | _logger.LogDebug("Token usage - Input: {InputTokens}, Output: {OutputTokens}, Total: {TotalTokens}", |
| | | 111 | | usage.InputTokenCount, usage.OutputTokenCount, usage.TotalTokenCount); |
| | | 112 | | |
| | | 113 | | // Add usage to tracker |
| | | 114 | | _tokenUsageTracker.AddUsage(_model, usage); |
| | | 115 | | |
| | | 116 | | // Calculate and log costs |
| | | 117 | | _costCalculationService.LogCostBreakdown(_model, usage); |
| | | 118 | | |
| | | 119 | | return prediction; |
| | | 120 | | } |
| | | 121 | | catch (Exception ex) |
| | | 122 | | { |
| | | 123 | | _logger.LogError(ex, "Error generating prediction for match: {HomeTeam} vs {AwayTeam}", |
| | | 124 | | match.HomeTeam, match.AwayTeam); |
| | | 125 | | Console.Error.WriteLine($"Prediction error for {match.HomeTeam} vs {match.AwayTeam}: {ex.Message}"); |
| | | 126 | | |
| | | 127 | | return null; |
| | | 128 | | } |
| | | 129 | | } |
| | | 130 | | |
| | | 131 | | public async Task<BonusPrediction?> PredictBonusQuestionAsync( |
| | | 132 | | BonusQuestion bonusQuestion, |
| | | 133 | | IEnumerable<DocumentContext> contextDocuments, |
| | | 134 | | CancellationToken cancellationToken = default) |
| | | 135 | | { |
| | | 136 | | _logger.LogInformation("Generating prediction for bonus question: {QuestionText}", bonusQuestion.Text); |
| | | 137 | | |
| | | 138 | | try |
| | | 139 | | { |
| | | 140 | | // Build the instructions by combining template with context |
| | | 141 | | var instructions = BuildBonusInstructions(contextDocuments); |
| | | 142 | | |
| | | 143 | | // Create bonus question JSON |
| | | 144 | | var questionJson = CreateBonusQuestionJson(bonusQuestion); |
| | | 145 | | |
| | | 146 | | _logger.LogDebug("Instructions length: {InstructionsLength} characters", instructions.Length); |
| | | 147 | | _logger.LogDebug("Context documents: {ContextCount}", contextDocuments.Count()); |
| | | 148 | | _logger.LogDebug("Question JSON: {QuestionJson}", questionJson); |
| | | 149 | | |
| | | 150 | | // Create messages for the chat completion |
| | | 151 | | var messages = new List<ChatMessage> |
| | | 152 | | { |
| | | 153 | | new SystemChatMessage(instructions), |
| | | 154 | | new UserChatMessage(questionJson) |
| | | 155 | | }; |
| | | 156 | | |
| | | 157 | | _logger.LogDebug("Calling OpenAI API for bonus prediction"); |
| | | 158 | | |
| | | 159 | | // Create JSON schema based on the question |
| | | 160 | | var jsonSchema = CreateSingleBonusPredictionJsonSchema(bonusQuestion); |
| | | 161 | | |
| | | 162 | | // Call OpenAI with structured output format |
| | | 163 | | var response = await _chatClient.CompleteChatAsync( |
| | | 164 | | messages, |
| | | 165 | | new ChatCompletionOptions |
| | | 166 | | { |
| | | 167 | | MaxOutputTokenCount = 10_000, // Standard limit for single question |
| | | 168 | | ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat( |
| | | 169 | | jsonSchemaFormatName: "bonus_prediction", |
| | | 170 | | jsonSchema: BinaryData.FromBytes(jsonSchema), |
| | | 171 | | jsonSchemaIsStrict: true) |
| | | 172 | | }, |
| | | 173 | | cancellationToken); |
| | | 174 | | |
| | | 175 | | // Parse the structured response |
| | | 176 | | var predictionJson = response.Value.Content[0].Text; |
| | | 177 | | _logger.LogDebug("Received bonus prediction JSON: {PredictionJson}", predictionJson); |
| | | 178 | | |
| | | 179 | | var prediction = ParseSingleBonusPrediction(predictionJson, bonusQuestion); |
| | | 180 | | |
| | | 181 | | if (prediction != null) |
| | | 182 | | { |
| | | 183 | | _logger.LogInformation("Generated prediction for bonus question: {SelectedOptions}", |
| | | 184 | | string.Join(", ", prediction.SelectedOptionIds)); |
| | | 185 | | } |
| | | 186 | | |
| | | 187 | | // Log token usage and cost breakdown |
| | | 188 | | var usage = response.Value.Usage; |
| | | 189 | | _logger.LogDebug("Token usage - Input: {InputTokens}, Output: {OutputTokens}, Total: {TotalTokens}", |
| | | 190 | | usage.InputTokenCount, usage.OutputTokenCount, usage.TotalTokenCount); |
| | | 191 | | |
| | | 192 | | // Add usage to tracker |
| | | 193 | | _tokenUsageTracker.AddUsage(_model, usage); |
| | | 194 | | |
| | | 195 | | // Calculate and log costs |
| | | 196 | | _costCalculationService.LogCostBreakdown(_model, usage); |
| | | 197 | | |
| | | 198 | | return prediction; |
| | | 199 | | } |
| | | 200 | | catch (Exception ex) |
| | | 201 | | { |
| | | 202 | | _logger.LogError(ex, "Error generating bonus prediction for question: {QuestionText}", bonusQuestion.Text); |
| | | 203 | | return null; |
| | | 204 | | } |
| | | 205 | | } |
| | | 206 | | |
| | | 207 | | private string BuildInstructions(IEnumerable<DocumentContext> contextDocuments, bool includeJustification) |
| | | 208 | | { |
| | | 209 | | var instructions = includeJustification |
| | | 210 | | ? _instructionsTemplateWithJustification |
| | | 211 | | : _instructionsTemplate; |
| | | 212 | | |
| | | 213 | | var contextList = contextDocuments.ToList(); |
| | | 214 | | if (contextList.Any()) |
| | | 215 | | { |
| | | 216 | | var contextSection = "\n"; |
| | | 217 | | foreach (var doc in contextList) |
| | | 218 | | { |
| | | 219 | | contextSection += "---\n"; |
| | | 220 | | contextSection += $"{doc.Name}\n\n"; |
| | | 221 | | contextSection += $"{doc.Content}\n"; |
| | | 222 | | } |
| | | 223 | | contextSection += "---"; |
| | | 224 | | |
| | | 225 | | instructions += contextSection; |
| | | 226 | | |
| | | 227 | | _logger.LogDebug("Added {ContextCount} context documents to instructions", contextList.Count); |
| | | 228 | | } |
| | | 229 | | else |
| | | 230 | | { |
| | | 231 | | _logger.LogDebug("No context documents provided"); |
| | | 232 | | } |
| | | 233 | | |
| | | 234 | | return instructions; |
| | | 235 | | } |
| | | 236 | | |
| | | 237 | | private static string CreateMatchJson(Match match) |
| | | 238 | | { |
| | | 239 | | return JsonSerializer.Serialize(new |
| | | 240 | | { |
| | | 241 | | homeTeam = match.HomeTeam, |
| | | 242 | | awayTeam = match.AwayTeam, |
| | | 243 | | startsAt = match.StartsAt.ToString() |
| | | 244 | | }, new JsonSerializerOptions |
| | | 245 | | { |
| | | 246 | | Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping |
| | | 247 | | }); |
| | | 248 | | } |
| | | 249 | | |
| | | 250 | | private static byte[] BuildPredictionJsonSchema(bool includeJustification) |
| | | 251 | | { |
| | | 252 | | var properties = new Dictionary<string, object?> |
| | | 253 | | { |
| | | 254 | | ["home"] = new Dictionary<string, object?> |
| | | 255 | | { |
| | | 256 | | ["type"] = "integer", |
| | | 257 | | ["description"] = "Predicted goals for the home team" |
| | | 258 | | }, |
| | | 259 | | ["away"] = new Dictionary<string, object?> |
| | | 260 | | { |
| | | 261 | | ["type"] = "integer", |
| | | 262 | | ["description"] = "Predicted goals for the away team" |
| | | 263 | | } |
| | | 264 | | }; |
| | | 265 | | |
| | | 266 | | var required = new List<string> { "home", "away" }; |
| | | 267 | | |
| | | 268 | | if (includeJustification) |
| | | 269 | | { |
| | | 270 | | var mostValuableContextSourceItem = new Dictionary<string, object?> |
| | | 271 | | { |
| | | 272 | | ["type"] = "object", |
| | | 273 | | ["properties"] = new Dictionary<string, object?> |
| | | 274 | | { |
| | | 275 | | ["documentName"] = new Dictionary<string, object?> |
| | | 276 | | { |
| | | 277 | | ["type"] = "string", |
| | | 278 | | ["description"] = "Name of the context document referenced" |
| | | 279 | | }, |
| | | 280 | | ["details"] = new Dictionary<string, object?> |
| | | 281 | | { |
| | | 282 | | ["type"] = "string", |
| | | 283 | | ["description"] = "Brief summary of why the document or parts of it were useful" |
| | | 284 | | } |
| | | 285 | | }, |
| | | 286 | | ["required"] = new[] { "documentName", "details" }, |
| | | 287 | | ["additionalProperties"] = false |
| | | 288 | | }; |
| | | 289 | | |
| | | 290 | | var leastValuableContextSourceItem = new Dictionary<string, object?> |
| | | 291 | | { |
| | | 292 | | ["type"] = "object", |
| | | 293 | | ["properties"] = new Dictionary<string, object?> |
| | | 294 | | { |
| | | 295 | | ["documentName"] = new Dictionary<string, object?> |
| | | 296 | | { |
| | | 297 | | ["type"] = "string", |
| | | 298 | | ["description"] = "Name of the context document referenced" |
| | | 299 | | }, |
| | | 300 | | ["details"] = new Dictionary<string, object?> |
| | | 301 | | { |
| | | 302 | | ["type"] = "string", |
| | | 303 | | ["description"] = "Brief summary explaining why the document or parts of it offered limited insi |
| | | 304 | | } |
| | | 305 | | }, |
| | | 306 | | ["required"] = new[] { "documentName", "details" }, |
| | | 307 | | ["additionalProperties"] = false |
| | | 308 | | }; |
| | | 309 | | |
| | | 310 | | var contextSources = new Dictionary<string, object?> |
| | | 311 | | { |
| | | 312 | | ["type"] = "object", |
| | | 313 | | ["properties"] = new Dictionary<string, object?> |
| | | 314 | | { |
| | | 315 | | ["mostValuable"] = new Dictionary<string, object?> |
| | | 316 | | { |
| | | 317 | | ["type"] = "array", |
| | | 318 | | ["items"] = mostValuableContextSourceItem, |
| | | 319 | | ["description"] = "Context documents that most influenced the prediction", |
| | | 320 | | ["minItems"] = 0 |
| | | 321 | | }, |
| | | 322 | | ["leastValuable"] = new Dictionary<string, object?> |
| | | 323 | | { |
| | | 324 | | ["type"] = "array", |
| | | 325 | | ["items"] = leastValuableContextSourceItem, |
| | | 326 | | ["description"] = "Context documents that provided limited or no valuable insight", |
| | | 327 | | ["minItems"] = 0 |
| | | 328 | | } |
| | | 329 | | }, |
| | | 330 | | ["required"] = new[] { "leastValuable", "mostValuable" }, |
| | | 331 | | ["additionalProperties"] = false |
| | | 332 | | }; |
| | | 333 | | |
| | | 334 | | properties["justification"] = new Dictionary<string, object?> |
| | | 335 | | { |
| | | 336 | | ["type"] = "object", |
| | | 337 | | ["properties"] = new Dictionary<string, object?> |
| | | 338 | | { |
| | | 339 | | ["keyReasoning"] = new Dictionary<string, object?> |
| | | 340 | | { |
| | | 341 | | ["type"] = "string", |
| | | 342 | | ["description"] = "Concise analytic summary motivating the predicted scoreline" |
| | | 343 | | }, |
| | | 344 | | ["contextSources"] = contextSources, |
| | | 345 | | ["uncertainties"] = new Dictionary<string, object?> |
| | | 346 | | { |
| | | 347 | | ["type"] = "array", |
| | | 348 | | ["items"] = new Dictionary<string, object?> |
| | | 349 | | { |
| | | 350 | | ["type"] = "string", |
| | | 351 | | ["description"] = "Single uncertainty or external factor affecting confidence" |
| | | 352 | | }, |
| | | 353 | | ["description"] = "Factors that could alter the predicted outcome", |
| | | 354 | | ["minItems"] = 0 |
| | | 355 | | } |
| | | 356 | | }, |
| | | 357 | | ["required"] = new[] { "contextSources", "keyReasoning", "uncertainties" }, |
| | | 358 | | ["additionalProperties"] = false |
| | | 359 | | }; |
| | | 360 | | required.Add("justification"); |
| | | 361 | | } |
| | | 362 | | |
| | | 363 | | var schema = new Dictionary<string, object?> |
| | | 364 | | { |
| | | 365 | | ["type"] = "object", |
| | | 366 | | ["properties"] = properties, |
| | | 367 | | ["required"] = required, |
| | | 368 | | ["additionalProperties"] = false |
| | | 369 | | }; |
| | | 370 | | |
| | | 371 | | return JsonSerializer.SerializeToUtf8Bytes(schema); |
| | | 372 | | } |
| | | 373 | | |
| | | 374 | | private Prediction ParsePrediction(string predictionJson) |
| | | 375 | | { |
| | | 376 | | try |
| | | 377 | | { |
| | | 378 | | _logger.LogDebug("Parsing prediction JSON: {PredictionJson}", predictionJson); |
| | | 379 | | |
| | | 380 | | var predictionResponse = JsonSerializer.Deserialize<PredictionResponse>(predictionJson); |
| | | 381 | | if (predictionResponse == null) |
| | | 382 | | { |
| | | 383 | | LogRawModelResponse(predictionJson); |
| | | 384 | | throw new InvalidOperationException("Failed to deserialize prediction response"); |
| | | 385 | | } |
| | | 386 | | |
| | | 387 | | _logger.LogDebug("Parsed prediction response - Home: {Home}, Away: {Away}", predictionResponse.Home, predict |
| | | 388 | | |
| | | 389 | | PredictionJustification? justification = null; |
| | | 390 | | |
| | | 391 | | if (predictionResponse.Justification != null) |
| | | 392 | | { |
| | | 393 | | var justificationResponse = predictionResponse.Justification; |
| | | 394 | | |
| | | 395 | | var mostValuable = justificationResponse.ContextSources?.MostValuable? |
| | | 396 | | .Where(entry => entry != null) |
| | | 397 | | .Select(entry => new PredictionJustificationContextSource( |
| | | 398 | | entry!.DocumentName?.Trim() ?? string.Empty, |
| | | 399 | | entry.Details?.Trim() ?? string.Empty)) |
| | | 400 | | .ToList() ?? new List<PredictionJustificationContextSource>(); |
| | | 401 | | |
| | | 402 | | var leastValuable = justificationResponse.ContextSources?.LeastValuable? |
| | | 403 | | .Where(entry => entry != null) |
| | | 404 | | .Select(entry => new PredictionJustificationContextSource( |
| | | 405 | | entry!.DocumentName?.Trim() ?? string.Empty, |
| | | 406 | | entry.Details?.Trim() ?? string.Empty)) |
| | | 407 | | .ToList() ?? new List<PredictionJustificationContextSource>(); |
| | | 408 | | |
| | | 409 | | var uncertainties = justificationResponse.Uncertainties? |
| | | 410 | | .Where(item => !string.IsNullOrWhiteSpace(item)) |
| | | 411 | | .Select(item => item.Trim()) |
| | | 412 | | .ToList() ?? new List<string>(); |
| | | 413 | | |
| | | 414 | | justification = new PredictionJustification( |
| | | 415 | | justificationResponse.KeyReasoning?.Trim() ?? string.Empty, |
| | | 416 | | new PredictionJustificationContextSources(mostValuable, leastValuable), |
| | | 417 | | uncertainties); |
| | | 418 | | |
| | | 419 | | _logger.LogDebug( |
| | | 420 | | "Parsed justification with key reasoning: {KeyReasoning}; Most valuable sources: {MostValuableCount} |
| | | 421 | | justification.KeyReasoning, |
| | | 422 | | justification.ContextSources.MostValuable.Count, |
| | | 423 | | justification.ContextSources.LeastValuable.Count, |
| | | 424 | | justification.Uncertainties.Count); |
| | | 425 | | } |
| | | 426 | | |
| | | 427 | | return new Prediction(predictionResponse.Home, predictionResponse.Away, justification); |
| | | 428 | | } |
| | | 429 | | catch (JsonException ex) |
| | | 430 | | { |
| | | 431 | | _logger.LogError(ex, "Failed to parse prediction JSON: {PredictionJson}", predictionJson); |
| | | 432 | | LogRawModelResponse(predictionJson); |
| | | 433 | | throw new InvalidOperationException($"Failed to parse prediction response: {ex.Message}", ex); |
| | | 434 | | } |
| | | 435 | | } |
| | | 436 | | |
| | | 437 | | private void LogRawModelResponse(string rawResponse) |
| | | 438 | | { |
| | | 439 | | if (string.IsNullOrWhiteSpace(rawResponse)) |
| | | 440 | | { |
| | | 441 | | const string message = "Raw model response from OpenAI was empty or whitespace."; |
| | | 442 | | _logger.LogError(message); |
| | | 443 | | Console.Error.WriteLine(message); |
| | | 444 | | return; |
| | | 445 | | } |
| | | 446 | | |
| | | 447 | | _logger.LogError("Raw model response from OpenAI: {RawResponse}", rawResponse); |
| | | 448 | | Console.Error.WriteLine("Raw model response from OpenAI:"); |
| | | 449 | | Console.Error.WriteLine(rawResponse); |
| | | 450 | | } |
| | | 451 | | |
| | | 452 | | private string BuildBonusInstructions(IEnumerable<DocumentContext> contextDocuments) |
| | | 453 | | { |
| | | 454 | | // Use the pre-loaded bonus instructions template |
| | | 455 | | var bonusInstructionsTemplate = _bonusInstructionsTemplate; |
| | | 456 | | |
| | | 457 | | var contextList = contextDocuments.ToList(); |
| | | 458 | | if (contextList.Any()) |
| | | 459 | | { |
| | | 460 | | var contextSection = "\n"; |
| | | 461 | | foreach (var doc in contextList) |
| | | 462 | | { |
| | | 463 | | contextSection += "---\n"; |
| | | 464 | | contextSection += $"{doc.Name}\n\n"; |
| | | 465 | | contextSection += $"{doc.Content}\n"; |
| | | 466 | | } |
| | | 467 | | contextSection += "---"; |
| | | 468 | | |
| | | 469 | | bonusInstructionsTemplate += contextSection; |
| | | 470 | | |
| | | 471 | | _logger.LogDebug("Added {ContextCount} context documents to bonus instructions", contextList.Count); |
| | | 472 | | } |
| | | 473 | | else |
| | | 474 | | { |
| | | 475 | | _logger.LogDebug("No context documents provided for bonus predictions"); |
| | | 476 | | } |
| | | 477 | | |
| | | 478 | | return bonusInstructionsTemplate; |
| | | 479 | | } |
| | | 480 | | |
| | | 481 | | private static string CreateBonusQuestionJson(BonusQuestion question) |
| | | 482 | | { |
| | | 483 | | var questionData = new |
| | | 484 | | { |
| | | 485 | | text = question.Text, |
| | | 486 | | options = question.Options.Select(o => new { id = o.Id, text = o.Text }).ToArray(), |
| | | 487 | | maxSelections = question.MaxSelections |
| | | 488 | | }; |
| | | 489 | | |
| | | 490 | | return JsonSerializer.Serialize(questionData, new JsonSerializerOptions |
| | | 491 | | { |
| | | 492 | | Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping |
| | | 493 | | }); |
| | | 494 | | } |
| | | 495 | | |
| | | 496 | | private static byte[] CreateSingleBonusPredictionJsonSchema(BonusQuestion question) |
| | | 497 | | { |
| | | 498 | | // For multi-selection questions, require exactly MaxSelections answers |
| | | 499 | | // For single-selection questions, require exactly 1 answer |
| | | 500 | | var requiredSelections = question.MaxSelections; |
| | | 501 | | |
| | | 502 | | var schema = new |
| | | 503 | | { |
| | | 504 | | type = "object", |
| | | 505 | | properties = new |
| | | 506 | | { |
| | | 507 | | selectedOptionIds = new |
| | | 508 | | { |
| | | 509 | | type = "array", |
| | | 510 | | items = new { type = "string", @enum = question.Options.Select(o => o.Id).ToArray() }, |
| | | 511 | | minItems = requiredSelections, |
| | | 512 | | maxItems = requiredSelections |
| | | 513 | | } |
| | | 514 | | }, |
| | | 515 | | required = new[] { "selectedOptionIds" }, |
| | | 516 | | additionalProperties = false |
| | | 517 | | }; |
| | | 518 | | |
| | | 519 | | return JsonSerializer.SerializeToUtf8Bytes(schema); |
| | | 520 | | } |
| | | 521 | | |
| | | 522 | | private BonusPrediction? ParseSingleBonusPrediction(string predictionJson, BonusQuestion question) |
| | | 523 | | { |
| | | 524 | | try |
| | | 525 | | { |
| | | 526 | | _logger.LogDebug("Parsing single bonus prediction JSON: {PredictionJson}", predictionJson); |
| | | 527 | | |
| | | 528 | | var response = JsonSerializer.Deserialize<SingleBonusPredictionResponse>(predictionJson); |
| | | 529 | | if (response?.SelectedOptionIds?.Any() != true) |
| | | 530 | | { |
| | | 531 | | throw new InvalidOperationException("Failed to deserialize bonus prediction response or no options selec |
| | | 532 | | } |
| | | 533 | | |
| | | 534 | | // Validate that all selected options exist for this question |
| | | 535 | | var validOptionIds = question.Options.Select(o => o.Id).ToHashSet(); |
| | | 536 | | var invalidOptions = response.SelectedOptionIds.Where(id => !validOptionIds.Contains(id)).ToArray(); |
| | | 537 | | |
| | | 538 | | if (invalidOptions.Any()) |
| | | 539 | | { |
| | | 540 | | _logger.LogWarning("Invalid option IDs for question '{QuestionText}': {InvalidOptions}", |
| | | 541 | | question.Text, string.Join(", ", invalidOptions)); |
| | | 542 | | return null; |
| | | 543 | | } |
| | | 544 | | |
| | | 545 | | // Validate no duplicate selections |
| | | 546 | | var duplicateOptions = response.SelectedOptionIds |
| | | 547 | | .GroupBy(id => id) |
| | | 548 | | .Where(g => g.Count() > 1) |
| | | 549 | | .Select(g => g.Key) |
| | | 550 | | .ToArray(); |
| | | 551 | | |
| | | 552 | | if (duplicateOptions.Any()) |
| | | 553 | | { |
| | | 554 | | _logger.LogWarning("Duplicate option IDs for question '{QuestionText}': {DuplicateOptions}", |
| | | 555 | | question.Text, string.Join(", ", duplicateOptions)); |
| | | 556 | | return null; |
| | | 557 | | } |
| | | 558 | | |
| | | 559 | | // Validate selection count - must match exactly MaxSelections for full predictions |
| | | 560 | | if (response.SelectedOptionIds.Length != question.MaxSelections) |
| | | 561 | | { |
| | | 562 | | _logger.LogWarning("Invalid selection count for question '{QuestionText}': expected exactly {MaxSelectio |
| | | 563 | | question.Text, question.MaxSelections, response.SelectedOptionIds.Length); |
| | | 564 | | return null; |
| | | 565 | | } |
| | | 566 | | |
| | | 567 | | var prediction = new BonusPrediction(response.SelectedOptionIds.ToList()); |
| | | 568 | | |
| | | 569 | | _logger.LogDebug("Parsed prediction: {SelectedOptions}", |
| | | 570 | | string.Join(", ", response.SelectedOptionIds)); |
| | | 571 | | |
| | | 572 | | return prediction; |
| | | 573 | | } |
| | | 574 | | catch (JsonException ex) |
| | | 575 | | { |
| | | 576 | | _logger.LogError(ex, "Failed to parse bonus prediction JSON: {PredictionJson}", predictionJson); |
| | | 577 | | return null; |
| | | 578 | | } |
| | | 579 | | } |
| | | 580 | | |
| | | 581 | | /// <summary> |
| | | 582 | | /// Gets the file path of the match prediction prompt being used by this service |
| | | 583 | | /// </summary> |
| | | 584 | | /// <returns>The absolute file path to the match prompt file</returns> |
| | | 585 | | public string GetMatchPromptPath(bool includeJustification = false) => includeJustification ? _matchPromptPathWithJu |
| | | 586 | | |
| | | 587 | | /// <summary> |
| | | 588 | | /// Gets the file path of the bonus question prediction prompt being used by this service |
| | | 589 | | /// </summary> |
| | | 590 | | /// <returns>The absolute file path to the bonus prompt file</returns> |
| | | 591 | | public string GetBonusPromptPath() => _bonusPromptPath; |
| | | 592 | | |
| | | 593 | | /// <summary> |
| | | 594 | | /// Internal class for deserializing the structured prediction response |
| | | 595 | | /// </summary> |
| | | 596 | | private class PredictionResponse |
| | | 597 | | { |
| | | 598 | | [JsonPropertyName("home")] |
| | 1 | 599 | | public int Home { get; set; } |
| | | 600 | | |
| | | 601 | | [JsonPropertyName("away")] |
| | 1 | 602 | | public int Away { get; set; } |
| | | 603 | | |
| | | 604 | | [JsonPropertyName("justification")] |
| | 1 | 605 | | public JustificationResponse? Justification { get; set; } |
| | | 606 | | } |
| | | 607 | | |
| | | 608 | | private class JustificationResponse |
| | | 609 | | { |
| | | 610 | | [JsonPropertyName("keyReasoning")] |
| | | 611 | | public string KeyReasoning { get; set; } = string.Empty; |
| | | 612 | | |
| | | 613 | | [JsonPropertyName("contextSources")] |
| | | 614 | | public JustificationContextSourcesResponse ContextSources { get; set; } = new(); |
| | | 615 | | |
| | | 616 | | [JsonPropertyName("uncertainties")] |
| | | 617 | | public string[] Uncertainties { get; set; } = Array.Empty<string>(); |
| | | 618 | | } |
| | | 619 | | |
| | | 620 | | private class JustificationContextSourcesResponse |
| | | 621 | | { |
| | | 622 | | [JsonPropertyName("mostValuable")] |
| | | 623 | | public JustificationContextSourceEntry[] MostValuable { get; set; } = Array.Empty<JustificationContextSourceEntr |
| | | 624 | | |
| | | 625 | | [JsonPropertyName("leastValuable")] |
| | | 626 | | public JustificationContextSourceEntry[] LeastValuable { get; set; } = Array.Empty<JustificationContextSourceEnt |
| | | 627 | | } |
| | | 628 | | |
| | | 629 | | private class JustificationContextSourceEntry |
| | | 630 | | { |
| | | 631 | | [JsonPropertyName("documentName")] |
| | | 632 | | public string DocumentName { get; set; } = string.Empty; |
| | | 633 | | |
| | | 634 | | [JsonPropertyName("details")] |
| | | 635 | | public string Details { get; set; } = string.Empty; |
| | | 636 | | } |
| | | 637 | | |
| | | 638 | | /// <summary> |
| | | 639 | | /// Internal class for deserializing the bonus predictions response |
| | | 640 | | /// </summary> |
| | | 641 | | private class BonusPredictionsResponse |
| | | 642 | | { |
| | | 643 | | [JsonPropertyName("predictions")] |
| | | 644 | | public BonusPredictionEntry[]? Predictions { get; set; } |
| | | 645 | | } |
| | | 646 | | |
| | | 647 | | /// <summary> |
| | | 648 | | /// Internal class for deserializing individual bonus prediction entries |
| | | 649 | | /// </summary> |
| | | 650 | | private class BonusPredictionEntry |
| | | 651 | | { |
| | | 652 | | [JsonPropertyName("questionId")] |
| | | 653 | | public string QuestionId { get; set; } = string.Empty; |
| | | 654 | | |
| | | 655 | | [JsonPropertyName("selectedOptionIds")] |
| | | 656 | | public string[] SelectedOptionIds { get; set; } = Array.Empty<string>(); |
| | | 657 | | } |
| | | 658 | | |
| | | 659 | | /// <summary> |
| | | 660 | | /// Internal class for deserializing single bonus prediction response |
| | | 661 | | /// </summary> |
| | | 662 | | private class SingleBonusPredictionResponse |
| | | 663 | | { |
| | | 664 | | [JsonPropertyName("selectedOptionIds")] |
| | | 665 | | public string[] SelectedOptionIds { get; set; } = Array.Empty<string>(); |
| | | 666 | | } |
| | | 667 | | } |