| | | 1 | | using System.Collections.Generic; |
| | | 2 | | using System.Diagnostics; |
| | | 3 | | using System.Linq; |
| | | 4 | | using System.Text.Json; |
| | | 5 | | using System.Text.Json.Serialization; |
| | | 6 | | using EHonda.KicktippAi.Core; |
| | | 7 | | using Microsoft.Extensions.Logging; |
| | | 8 | | using OpenAI.Chat; |
| | | 9 | | |
| | | 10 | | namespace OpenAiIntegration; |
| | | 11 | | |
| | | 12 | | /// <summary> |
| | | 13 | | /// Service for predicting match outcomes using OpenAI models |
| | | 14 | | /// </summary> |
| | | 15 | | public class PredictionService : IPredictionService |
| | | 16 | | { |
| | | 17 | | private readonly ChatClient _chatClient; |
| | | 18 | | private readonly ILogger<PredictionService> _logger; |
| | | 19 | | private readonly ICostCalculationService _costCalculationService; |
| | | 20 | | private readonly ITokenUsageTracker _tokenUsageTracker; |
| | | 21 | | private readonly IInstructionsTemplateProvider _templateProvider; |
| | | 22 | | private readonly string _model; |
| | | 23 | | private readonly string _instructionsTemplate; |
| | | 24 | | private readonly string _instructionsTemplateWithJustification; |
| | | 25 | | private readonly string _bonusInstructionsTemplate; |
| | | 26 | | private readonly string _matchPromptPath; |
| | | 27 | | private readonly string _matchPromptPathWithJustification; |
| | | 28 | | private readonly string _bonusPromptPath; |
| | | 29 | | |
| | 1 | 30 | | public PredictionService( |
| | 1 | 31 | | ChatClient chatClient, |
| | 1 | 32 | | ILogger<PredictionService> logger, |
| | 1 | 33 | | ICostCalculationService costCalculationService, |
| | 1 | 34 | | ITokenUsageTracker tokenUsageTracker, |
| | 1 | 35 | | IInstructionsTemplateProvider templateProvider, |
| | 1 | 36 | | string model) |
| | | 37 | | { |
| | 1 | 38 | | _chatClient = chatClient ?? throw new ArgumentNullException(nameof(chatClient)); |
| | 1 | 39 | | _logger = logger ?? throw new ArgumentNullException(nameof(logger)); |
| | 1 | 40 | | _costCalculationService = costCalculationService ?? throw new ArgumentNullException(nameof(costCalculationServic |
| | 1 | 41 | | _tokenUsageTracker = tokenUsageTracker ?? throw new ArgumentNullException(nameof(tokenUsageTracker)); |
| | 1 | 42 | | _templateProvider = templateProvider ?? throw new ArgumentNullException(nameof(templateProvider)); |
| | 1 | 43 | | _model = model ?? throw new ArgumentNullException(nameof(model)); |
| | | 44 | | |
| | 1 | 45 | | var (matchTemplate, matchPath) = _templateProvider.LoadMatchTemplate(_model, includeJustification: false); |
| | 1 | 46 | | var (matchJustificationTemplate, matchJustificationPath) = _templateProvider.LoadMatchTemplate(_model, includeJu |
| | 1 | 47 | | var (bonusTemplate, bonusPath) = _templateProvider.LoadBonusTemplate(_model); |
| | | 48 | | |
| | 1 | 49 | | _instructionsTemplate = matchTemplate; |
| | 1 | 50 | | _instructionsTemplateWithJustification = matchJustificationTemplate; |
| | 1 | 51 | | _bonusInstructionsTemplate = bonusTemplate; |
| | 1 | 52 | | _matchPromptPath = matchPath; |
| | 1 | 53 | | _matchPromptPathWithJustification = matchJustificationPath; |
| | 1 | 54 | | _bonusPromptPath = bonusPath; |
| | 1 | 55 | | } |
| | | 56 | | |
| | | 57 | | public async Task<Prediction?> PredictMatchAsync( |
| | | 58 | | Match match, |
| | | 59 | | IEnumerable<DocumentContext> contextDocuments, |
| | | 60 | | bool includeJustification = false, |
| | | 61 | | PredictionTelemetryMetadata? telemetryMetadata = null, |
| | | 62 | | CancellationToken cancellationToken = default) |
| | | 63 | | { |
| | 1 | 64 | | _logger.LogInformation("Generating prediction for match: {HomeTeam} vs {AwayTeam} at {StartTime}", |
| | 1 | 65 | | match.HomeTeam, match.AwayTeam, match.StartsAt); |
| | | 66 | | |
| | | 67 | | try |
| | | 68 | | { |
| | | 69 | | // Build the instructions by combining template with context |
| | 1 | 70 | | var instructions = BuildInstructions(contextDocuments, includeJustification); |
| | | 71 | | |
| | | 72 | | // Create match JSON |
| | 1 | 73 | | var matchJson = PredictionPromptComposer.CreateMatchJson(match); |
| | | 74 | | |
| | 1 | 75 | | _logger.LogDebug("Instructions length: {InstructionsLength} characters", instructions.Length); |
| | 1 | 76 | | _logger.LogDebug("Context documents: {ContextCount}", contextDocuments.Count()); |
| | 1 | 77 | | _logger.LogDebug("Match JSON: {MatchJson}", matchJson); |
| | | 78 | | |
| | | 79 | | // Create messages for the chat completion |
| | 1 | 80 | | var messages = new List<ChatMessage> |
| | 1 | 81 | | { |
| | 1 | 82 | | new SystemChatMessage(instructions), |
| | 1 | 83 | | new UserChatMessage(matchJson) |
| | 1 | 84 | | }; |
| | | 85 | | |
| | 1 | 86 | | _logger.LogDebug("Calling OpenAI API for prediction"); |
| | | 87 | | |
| | | 88 | | // Start an OTel activity for Langfuse generation tracking |
| | 1 | 89 | | using var activity = Telemetry.Source.StartActivity("predict-match"); |
| | | 90 | | |
| | | 91 | | // Call OpenAI with structured output format |
| | 1 | 92 | | var response = await _chatClient.CompleteChatAsync( |
| | 1 | 93 | | messages, |
| | 1 | 94 | | new ChatCompletionOptions |
| | 1 | 95 | | { |
| | 1 | 96 | | MaxOutputTokenCount = 10_000, // Safeguard against high costs |
| | 1 | 97 | | ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat( |
| | 1 | 98 | | jsonSchemaFormatName: "match_prediction", |
| | 1 | 99 | | jsonSchema: BinaryData.FromBytes(BuildPredictionJsonSchema(includeJustification)), |
| | 1 | 100 | | jsonSchemaIsStrict: true) |
| | 1 | 101 | | }, |
| | 1 | 102 | | cancellationToken); |
| | | 103 | | |
| | | 104 | | // Parse the structured response |
| | 1 | 105 | | var predictionJson = response.Value.Content[0].Text; |
| | 1 | 106 | | _logger.LogDebug("Received prediction JSON: {PredictionJson}", predictionJson); |
| | | 107 | | |
| | 1 | 108 | | var prediction = ParsePrediction(predictionJson); |
| | | 109 | | |
| | 1 | 110 | | _logger.LogInformation("Prediction generated: {HomeGoals}-{AwayGoals} for {HomeTeam} vs {AwayTeam}", |
| | 1 | 111 | | prediction.HomeGoals, prediction.AwayGoals, match.HomeTeam, match.AwayTeam); |
| | | 112 | | |
| | | 113 | | // Log token usage and cost breakdown |
| | 1 | 114 | | var usage = response.Value.Usage; |
| | 1 | 115 | | _logger.LogDebug("Token usage - Input: {InputTokens}, Output: {OutputTokens}, Total: {TotalTokens}", |
| | 1 | 116 | | usage.InputTokenCount, usage.OutputTokenCount, usage.TotalTokenCount); |
| | | 117 | | |
| | | 118 | | // Set Langfuse generation attributes on the activity |
| | 1 | 119 | | SetLangfuseGenerationAttributes(activity, messages, predictionJson, usage, telemetryMetadata); |
| | | 120 | | |
| | | 121 | | // Add usage to tracker |
| | 1 | 122 | | _tokenUsageTracker.AddUsage(_model, usage); |
| | | 123 | | |
| | | 124 | | // Calculate and log costs |
| | 1 | 125 | | _costCalculationService.LogCostBreakdown(_model, usage); |
| | | 126 | | |
| | 1 | 127 | | return prediction; |
| | | 128 | | } |
| | 1 | 129 | | catch (Exception ex) |
| | | 130 | | { |
| | 1 | 131 | | _logger.LogError(ex, "Error generating prediction for match: {HomeTeam} vs {AwayTeam}", |
| | 1 | 132 | | match.HomeTeam, match.AwayTeam); |
| | 1 | 133 | | Console.Error.WriteLine($"Prediction error for {match.HomeTeam} vs {match.AwayTeam}: {ex.Message}"); |
| | | 134 | | |
| | 1 | 135 | | return null; |
| | | 136 | | } |
| | 1 | 137 | | } |
| | | 138 | | |
| | | 139 | | public async Task<BonusPrediction?> PredictBonusQuestionAsync( |
| | | 140 | | BonusQuestion bonusQuestion, |
| | | 141 | | IEnumerable<DocumentContext> contextDocuments, |
| | | 142 | | PredictionTelemetryMetadata? telemetryMetadata = null, |
| | | 143 | | CancellationToken cancellationToken = default) |
| | | 144 | | { |
| | 1 | 145 | | _logger.LogInformation("Generating prediction for bonus question: {QuestionText}", bonusQuestion.Text); |
| | | 146 | | |
| | | 147 | | try |
| | | 148 | | { |
| | | 149 | | // Build the instructions by combining template with context |
| | 1 | 150 | | var instructions = BuildBonusInstructions(contextDocuments); |
| | | 151 | | |
| | | 152 | | // Create bonus question JSON |
| | 1 | 153 | | var questionJson = PredictionPromptComposer.CreateBonusQuestionJson(bonusQuestion); |
| | | 154 | | |
| | 1 | 155 | | _logger.LogDebug("Instructions length: {InstructionsLength} characters", instructions.Length); |
| | 1 | 156 | | _logger.LogDebug("Context documents: {ContextCount}", contextDocuments.Count()); |
| | 1 | 157 | | _logger.LogDebug("Question JSON: {QuestionJson}", questionJson); |
| | | 158 | | |
| | | 159 | | // Create messages for the chat completion |
| | 1 | 160 | | var messages = new List<ChatMessage> |
| | 1 | 161 | | { |
| | 1 | 162 | | new SystemChatMessage(instructions), |
| | 1 | 163 | | new UserChatMessage(questionJson) |
| | 1 | 164 | | }; |
| | | 165 | | |
| | 1 | 166 | | _logger.LogDebug("Calling OpenAI API for bonus prediction"); |
| | | 167 | | |
| | | 168 | | // Create JSON schema based on the question |
| | 1 | 169 | | var jsonSchema = CreateSingleBonusPredictionJsonSchema(bonusQuestion); |
| | | 170 | | |
| | | 171 | | // Start an OTel activity for Langfuse generation tracking |
| | 1 | 172 | | using var activity = Telemetry.Source.StartActivity("predict-bonus"); |
| | | 173 | | |
| | | 174 | | // Call OpenAI with structured output format |
| | 1 | 175 | | var response = await _chatClient.CompleteChatAsync( |
| | 1 | 176 | | messages, |
| | 1 | 177 | | new ChatCompletionOptions |
| | 1 | 178 | | { |
| | 1 | 179 | | MaxOutputTokenCount = 10_000, // Standard limit for single question |
| | 1 | 180 | | ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat( |
| | 1 | 181 | | jsonSchemaFormatName: "bonus_prediction", |
| | 1 | 182 | | jsonSchema: BinaryData.FromBytes(jsonSchema), |
| | 1 | 183 | | jsonSchemaIsStrict: true) |
| | 1 | 184 | | }, |
| | 1 | 185 | | cancellationToken); |
| | | 186 | | |
| | | 187 | | // Parse the structured response |
| | 1 | 188 | | var predictionJson = response.Value.Content[0].Text; |
| | 1 | 189 | | _logger.LogDebug("Received bonus prediction JSON: {PredictionJson}", predictionJson); |
| | | 190 | | |
| | 1 | 191 | | var prediction = ParseSingleBonusPrediction(predictionJson, bonusQuestion); |
| | | 192 | | |
| | 1 | 193 | | if (prediction != null) |
| | | 194 | | { |
| | 1 | 195 | | _logger.LogInformation("Generated prediction for bonus question: {SelectedOptions}", |
| | 1 | 196 | | string.Join(", ", prediction.SelectedOptionIds)); |
| | | 197 | | } |
| | | 198 | | |
| | | 199 | | // Log token usage and cost breakdown |
| | 1 | 200 | | var usage = response.Value.Usage; |
| | 1 | 201 | | _logger.LogDebug("Token usage - Input: {InputTokens}, Output: {OutputTokens}, Total: {TotalTokens}", |
| | 1 | 202 | | usage.InputTokenCount, usage.OutputTokenCount, usage.TotalTokenCount); |
| | | 203 | | |
| | | 204 | | // Set Langfuse generation attributes on the activity |
| | 1 | 205 | | SetLangfuseGenerationAttributes(activity, messages, predictionJson, usage, telemetryMetadata); |
| | | 206 | | |
| | | 207 | | // Add usage to tracker |
| | 1 | 208 | | _tokenUsageTracker.AddUsage(_model, usage); |
| | | 209 | | |
| | | 210 | | // Calculate and log costs |
| | 1 | 211 | | _costCalculationService.LogCostBreakdown(_model, usage); |
| | | 212 | | |
| | 1 | 213 | | return prediction; |
| | | 214 | | } |
| | 1 | 215 | | catch (Exception ex) |
| | | 216 | | { |
| | 1 | 217 | | _logger.LogError(ex, "Error generating bonus prediction for question: {QuestionText}", bonusQuestion.Text); |
| | 1 | 218 | | return null; |
| | | 219 | | } |
| | 1 | 220 | | } |
| | | 221 | | |
| | | 222 | | private string BuildInstructions(IEnumerable<DocumentContext> contextDocuments, bool includeJustification) |
| | | 223 | | { |
| | 1 | 224 | | var template = includeJustification |
| | 1 | 225 | | ? _instructionsTemplateWithJustification |
| | 1 | 226 | | : _instructionsTemplate; |
| | | 227 | | |
| | 1 | 228 | | var contextList = contextDocuments.ToList(); |
| | 1 | 229 | | if (contextList.Any()) |
| | | 230 | | { |
| | 1 | 231 | | _logger.LogDebug("Added {ContextCount} context documents to instructions", contextList.Count); |
| | | 232 | | } |
| | | 233 | | else |
| | | 234 | | { |
| | 1 | 235 | | _logger.LogDebug("No context documents provided"); |
| | | 236 | | } |
| | | 237 | | |
| | 1 | 238 | | return PredictionPromptComposer.BuildSystemPrompt(template, contextList); |
| | | 239 | | } |
| | | 240 | | |
| | | 241 | | private static byte[] BuildPredictionJsonSchema(bool includeJustification) |
| | | 242 | | { |
| | 1 | 243 | | var properties = new Dictionary<string, object?> |
| | 1 | 244 | | { |
| | 1 | 245 | | ["home"] = new Dictionary<string, object?> |
| | 1 | 246 | | { |
| | 1 | 247 | | ["type"] = "integer", |
| | 1 | 248 | | ["description"] = "Predicted goals for the home team" |
| | 1 | 249 | | }, |
| | 1 | 250 | | ["away"] = new Dictionary<string, object?> |
| | 1 | 251 | | { |
| | 1 | 252 | | ["type"] = "integer", |
| | 1 | 253 | | ["description"] = "Predicted goals for the away team" |
| | 1 | 254 | | } |
| | 1 | 255 | | }; |
| | | 256 | | |
| | 1 | 257 | | var required = new List<string> { "home", "away" }; |
| | | 258 | | |
| | 1 | 259 | | if (includeJustification) |
| | | 260 | | { |
| | 1 | 261 | | var mostValuableContextSourceItem = new Dictionary<string, object?> |
| | 1 | 262 | | { |
| | 1 | 263 | | ["type"] = "object", |
| | 1 | 264 | | ["properties"] = new Dictionary<string, object?> |
| | 1 | 265 | | { |
| | 1 | 266 | | ["documentName"] = new Dictionary<string, object?> |
| | 1 | 267 | | { |
| | 1 | 268 | | ["type"] = "string", |
| | 1 | 269 | | ["description"] = "Name of the context document referenced" |
| | 1 | 270 | | }, |
| | 1 | 271 | | ["details"] = new Dictionary<string, object?> |
| | 1 | 272 | | { |
| | 1 | 273 | | ["type"] = "string", |
| | 1 | 274 | | ["description"] = "Brief summary of why the document or parts of it were useful" |
| | 1 | 275 | | } |
| | 1 | 276 | | }, |
| | 1 | 277 | | ["required"] = new[] { "documentName", "details" }, |
| | 1 | 278 | | ["additionalProperties"] = false |
| | 1 | 279 | | }; |
| | | 280 | | |
| | 1 | 281 | | var leastValuableContextSourceItem = new Dictionary<string, object?> |
| | 1 | 282 | | { |
| | 1 | 283 | | ["type"] = "object", |
| | 1 | 284 | | ["properties"] = new Dictionary<string, object?> |
| | 1 | 285 | | { |
| | 1 | 286 | | ["documentName"] = new Dictionary<string, object?> |
| | 1 | 287 | | { |
| | 1 | 288 | | ["type"] = "string", |
| | 1 | 289 | | ["description"] = "Name of the context document referenced" |
| | 1 | 290 | | }, |
| | 1 | 291 | | ["details"] = new Dictionary<string, object?> |
| | 1 | 292 | | { |
| | 1 | 293 | | ["type"] = "string", |
| | 1 | 294 | | ["description"] = "Brief summary explaining why the document or parts of it offered limited insi |
| | 1 | 295 | | } |
| | 1 | 296 | | }, |
| | 1 | 297 | | ["required"] = new[] { "documentName", "details" }, |
| | 1 | 298 | | ["additionalProperties"] = false |
| | 1 | 299 | | }; |
| | | 300 | | |
| | 1 | 301 | | var contextSources = new Dictionary<string, object?> |
| | 1 | 302 | | { |
| | 1 | 303 | | ["type"] = "object", |
| | 1 | 304 | | ["properties"] = new Dictionary<string, object?> |
| | 1 | 305 | | { |
| | 1 | 306 | | ["mostValuable"] = new Dictionary<string, object?> |
| | 1 | 307 | | { |
| | 1 | 308 | | ["type"] = "array", |
| | 1 | 309 | | ["items"] = mostValuableContextSourceItem, |
| | 1 | 310 | | ["description"] = "Context documents that most influenced the prediction", |
| | 1 | 311 | | ["minItems"] = 0 |
| | 1 | 312 | | }, |
| | 1 | 313 | | ["leastValuable"] = new Dictionary<string, object?> |
| | 1 | 314 | | { |
| | 1 | 315 | | ["type"] = "array", |
| | 1 | 316 | | ["items"] = leastValuableContextSourceItem, |
| | 1 | 317 | | ["description"] = "Context documents that provided limited or no valuable insight", |
| | 1 | 318 | | ["minItems"] = 0 |
| | 1 | 319 | | } |
| | 1 | 320 | | }, |
| | 1 | 321 | | ["required"] = new[] { "leastValuable", "mostValuable" }, |
| | 1 | 322 | | ["additionalProperties"] = false |
| | 1 | 323 | | }; |
| | | 324 | | |
| | 1 | 325 | | properties["justification"] = new Dictionary<string, object?> |
| | 1 | 326 | | { |
| | 1 | 327 | | ["type"] = "object", |
| | 1 | 328 | | ["properties"] = new Dictionary<string, object?> |
| | 1 | 329 | | { |
| | 1 | 330 | | ["keyReasoning"] = new Dictionary<string, object?> |
| | 1 | 331 | | { |
| | 1 | 332 | | ["type"] = "string", |
| | 1 | 333 | | ["description"] = "Concise analytic summary motivating the predicted scoreline" |
| | 1 | 334 | | }, |
| | 1 | 335 | | ["contextSources"] = contextSources, |
| | 1 | 336 | | ["uncertainties"] = new Dictionary<string, object?> |
| | 1 | 337 | | { |
| | 1 | 338 | | ["type"] = "array", |
| | 1 | 339 | | ["items"] = new Dictionary<string, object?> |
| | 1 | 340 | | { |
| | 1 | 341 | | ["type"] = "string", |
| | 1 | 342 | | ["description"] = "Single uncertainty or external factor affecting confidence" |
| | 1 | 343 | | }, |
| | 1 | 344 | | ["description"] = "Factors that could alter the predicted outcome", |
| | 1 | 345 | | ["minItems"] = 0 |
| | 1 | 346 | | } |
| | 1 | 347 | | }, |
| | 1 | 348 | | ["required"] = new[] { "contextSources", "keyReasoning", "uncertainties" }, |
| | 1 | 349 | | ["additionalProperties"] = false |
| | 1 | 350 | | }; |
| | 1 | 351 | | required.Add("justification"); |
| | | 352 | | } |
| | | 353 | | |
| | 1 | 354 | | var schema = new Dictionary<string, object?> |
| | 1 | 355 | | { |
| | 1 | 356 | | ["type"] = "object", |
| | 1 | 357 | | ["properties"] = properties, |
| | 1 | 358 | | ["required"] = required, |
| | 1 | 359 | | ["additionalProperties"] = false |
| | 1 | 360 | | }; |
| | | 361 | | |
| | 1 | 362 | | return JsonSerializer.SerializeToUtf8Bytes(schema); |
| | | 363 | | } |
| | | 364 | | |
| | | 365 | | private Prediction ParsePrediction(string predictionJson) |
| | | 366 | | { |
| | | 367 | | try |
| | | 368 | | { |
| | 1 | 369 | | _logger.LogDebug("Parsing prediction JSON: {PredictionJson}", predictionJson); |
| | | 370 | | |
| | 1 | 371 | | var predictionResponse = JsonSerializer.Deserialize<PredictionResponse>(predictionJson); |
| | 1 | 372 | | if (predictionResponse == null) |
| | | 373 | | { |
| | 0 | 374 | | LogRawModelResponse(predictionJson); |
| | 0 | 375 | | throw new InvalidOperationException("Failed to deserialize prediction response"); |
| | | 376 | | } |
| | | 377 | | |
| | 1 | 378 | | _logger.LogDebug("Parsed prediction response - Home: {Home}, Away: {Away}", predictionResponse.Home, predict |
| | | 379 | | |
| | 1 | 380 | | PredictionJustification? justification = null; |
| | | 381 | | |
| | 1 | 382 | | if (predictionResponse.Justification != null) |
| | | 383 | | { |
| | 1 | 384 | | var justificationResponse = predictionResponse.Justification; |
| | | 385 | | |
| | 1 | 386 | | var mostValuable = justificationResponse.ContextSources?.MostValuable? |
| | 1 | 387 | | .Where(entry => entry != null) |
| | 1 | 388 | | .Select(entry => new PredictionJustificationContextSource( |
| | 1 | 389 | | entry!.DocumentName?.Trim() ?? string.Empty, |
| | 1 | 390 | | entry.Details?.Trim() ?? string.Empty)) |
| | 1 | 391 | | .ToList() ?? new List<PredictionJustificationContextSource>(); |
| | | 392 | | |
| | 1 | 393 | | var leastValuable = justificationResponse.ContextSources?.LeastValuable? |
| | 0 | 394 | | .Where(entry => entry != null) |
| | 0 | 395 | | .Select(entry => new PredictionJustificationContextSource( |
| | 0 | 396 | | entry!.DocumentName?.Trim() ?? string.Empty, |
| | 0 | 397 | | entry.Details?.Trim() ?? string.Empty)) |
| | 1 | 398 | | .ToList() ?? new List<PredictionJustificationContextSource>(); |
| | | 399 | | |
| | 1 | 400 | | var uncertainties = justificationResponse.Uncertainties? |
| | 1 | 401 | | .Where(item => !string.IsNullOrWhiteSpace(item)) |
| | 1 | 402 | | .Select(item => item.Trim()) |
| | 1 | 403 | | .ToList() ?? new List<string>(); |
| | | 404 | | |
| | 1 | 405 | | justification = new PredictionJustification( |
| | 1 | 406 | | justificationResponse.KeyReasoning?.Trim() ?? string.Empty, |
| | 1 | 407 | | new PredictionJustificationContextSources(mostValuable, leastValuable), |
| | 1 | 408 | | uncertainties); |
| | | 409 | | |
| | 1 | 410 | | _logger.LogDebug( |
| | 1 | 411 | | "Parsed justification with key reasoning: {KeyReasoning}; Most valuable sources: {MostValuableCount} |
| | 1 | 412 | | justification.KeyReasoning, |
| | 1 | 413 | | justification.ContextSources.MostValuable.Count, |
| | 1 | 414 | | justification.ContextSources.LeastValuable.Count, |
| | 1 | 415 | | justification.Uncertainties.Count); |
| | | 416 | | } |
| | | 417 | | |
| | 1 | 418 | | return new Prediction(predictionResponse.Home, predictionResponse.Away, justification); |
| | | 419 | | } |
| | 1 | 420 | | catch (JsonException ex) |
| | | 421 | | { |
| | 1 | 422 | | _logger.LogError(ex, "Failed to parse prediction JSON: {PredictionJson}", predictionJson); |
| | 1 | 423 | | LogRawModelResponse(predictionJson); |
| | 1 | 424 | | throw new InvalidOperationException($"Failed to parse prediction response: {ex.Message}", ex); |
| | | 425 | | } |
| | 1 | 426 | | } |
| | | 427 | | |
| | | 428 | | private void LogRawModelResponse(string rawResponse) |
| | | 429 | | { |
| | 1 | 430 | | if (string.IsNullOrWhiteSpace(rawResponse)) |
| | | 431 | | { |
| | | 432 | | const string message = "Raw model response from OpenAI was empty or whitespace."; |
| | 0 | 433 | | _logger.LogError(message); |
| | 0 | 434 | | Console.Error.WriteLine(message); |
| | 0 | 435 | | return; |
| | | 436 | | } |
| | | 437 | | |
| | 1 | 438 | | _logger.LogError("Raw model response from OpenAI: {RawResponse}", rawResponse); |
| | 1 | 439 | | Console.Error.WriteLine("Raw model response from OpenAI:"); |
| | 1 | 440 | | Console.Error.WriteLine(rawResponse); |
| | 1 | 441 | | } |
| | | 442 | | |
| | | 443 | | private string BuildBonusInstructions(IEnumerable<DocumentContext> contextDocuments) |
| | | 444 | | { |
| | | 445 | | // Use the pre-loaded bonus instructions template |
| | 1 | 446 | | var bonusInstructionsTemplate = _bonusInstructionsTemplate; |
| | | 447 | | |
| | 1 | 448 | | var contextList = contextDocuments.ToList(); |
| | 1 | 449 | | if (contextList.Any()) |
| | | 450 | | { |
| | 1 | 451 | | _logger.LogDebug("Added {ContextCount} context documents to bonus instructions", contextList.Count); |
| | | 452 | | } |
| | | 453 | | else |
| | | 454 | | { |
| | 1 | 455 | | _logger.LogDebug("No context documents provided for bonus predictions"); |
| | | 456 | | } |
| | | 457 | | |
| | 1 | 458 | | return PredictionPromptComposer.BuildSystemPrompt(bonusInstructionsTemplate, contextList); |
| | | 459 | | } |
| | | 460 | | |
| | | 461 | | private static byte[] CreateSingleBonusPredictionJsonSchema(BonusQuestion question) |
| | | 462 | | { |
| | | 463 | | // For multi-selection questions, require exactly MaxSelections answers |
| | | 464 | | // For single-selection questions, require exactly 1 answer |
| | 1 | 465 | | var requiredSelections = question.MaxSelections; |
| | | 466 | | |
| | 1 | 467 | | var schema = new |
| | 1 | 468 | | { |
| | 1 | 469 | | type = "object", |
| | 1 | 470 | | properties = new |
| | 1 | 471 | | { |
| | 1 | 472 | | selectedOptionIds = new |
| | 1 | 473 | | { |
| | 1 | 474 | | type = "array", |
| | 1 | 475 | | items = new { type = "string", @enum = question.Options.Select(o => o.Id).ToArray() }, |
| | 1 | 476 | | minItems = requiredSelections, |
| | 1 | 477 | | maxItems = requiredSelections |
| | 1 | 478 | | } |
| | 1 | 479 | | }, |
| | 1 | 480 | | required = new[] { "selectedOptionIds" }, |
| | 1 | 481 | | additionalProperties = false |
| | 1 | 482 | | }; |
| | | 483 | | |
| | 1 | 484 | | return JsonSerializer.SerializeToUtf8Bytes(schema); |
| | | 485 | | } |
| | | 486 | | |
| | | 487 | | private BonusPrediction? ParseSingleBonusPrediction(string predictionJson, BonusQuestion question) |
| | | 488 | | { |
| | | 489 | | try |
| | | 490 | | { |
| | 1 | 491 | | _logger.LogDebug("Parsing single bonus prediction JSON: {PredictionJson}", predictionJson); |
| | | 492 | | |
| | 1 | 493 | | var response = JsonSerializer.Deserialize<SingleBonusPredictionResponse>(predictionJson); |
| | 1 | 494 | | if (response?.SelectedOptionIds?.Any() != true) |
| | | 495 | | { |
| | 1 | 496 | | throw new InvalidOperationException("Failed to deserialize bonus prediction response or no options selec |
| | | 497 | | } |
| | | 498 | | |
| | | 499 | | // Validate that all selected options exist for this question |
| | 1 | 500 | | var validOptionIds = question.Options.Select(o => o.Id).ToHashSet(); |
| | 1 | 501 | | var invalidOptions = response.SelectedOptionIds.Where(id => !validOptionIds.Contains(id)).ToArray(); |
| | | 502 | | |
| | 1 | 503 | | if (invalidOptions.Any()) |
| | | 504 | | { |
| | 1 | 505 | | _logger.LogWarning("Invalid option IDs for question '{QuestionText}': {InvalidOptions}", |
| | 1 | 506 | | question.Text, string.Join(", ", invalidOptions)); |
| | 1 | 507 | | return null; |
| | | 508 | | } |
| | | 509 | | |
| | | 510 | | // Validate no duplicate selections |
| | 1 | 511 | | var duplicateOptions = response.SelectedOptionIds |
| | 1 | 512 | | .GroupBy(id => id) |
| | 1 | 513 | | .Where(g => g.Count() > 1) |
| | 1 | 514 | | .Select(g => g.Key) |
| | 1 | 515 | | .ToArray(); |
| | | 516 | | |
| | 1 | 517 | | if (duplicateOptions.Any()) |
| | | 518 | | { |
| | 1 | 519 | | _logger.LogWarning("Duplicate option IDs for question '{QuestionText}': {DuplicateOptions}", |
| | 1 | 520 | | question.Text, string.Join(", ", duplicateOptions)); |
| | 1 | 521 | | return null; |
| | | 522 | | } |
| | | 523 | | |
| | | 524 | | // Validate selection count - must match exactly MaxSelections for full predictions |
| | 1 | 525 | | if (response.SelectedOptionIds.Length != question.MaxSelections) |
| | | 526 | | { |
| | 1 | 527 | | _logger.LogWarning("Invalid selection count for question '{QuestionText}': expected exactly {MaxSelectio |
| | 1 | 528 | | question.Text, question.MaxSelections, response.SelectedOptionIds.Length); |
| | 1 | 529 | | return null; |
| | | 530 | | } |
| | | 531 | | |
| | 1 | 532 | | var prediction = new BonusPrediction(response.SelectedOptionIds.ToList()); |
| | | 533 | | |
| | 1 | 534 | | _logger.LogDebug("Parsed prediction: {SelectedOptions}", |
| | 1 | 535 | | string.Join(", ", response.SelectedOptionIds)); |
| | | 536 | | |
| | 1 | 537 | | return prediction; |
| | | 538 | | } |
| | 1 | 539 | | catch (JsonException ex) |
| | | 540 | | { |
| | 1 | 541 | | _logger.LogError(ex, "Failed to parse bonus prediction JSON: {PredictionJson}", predictionJson); |
| | 1 | 542 | | return null; |
| | | 543 | | } |
| | 1 | 544 | | } |
| | | 545 | | |
| | | 546 | | /// <summary> |
| | | 547 | | /// Gets the file path of the match prediction prompt being used by this service |
| | | 548 | | /// </summary> |
| | | 549 | | /// <returns>The absolute file path to the match prompt file</returns> |
| | 1 | 550 | | public string GetMatchPromptPath(bool includeJustification = false) => includeJustification ? _matchPromptPathWithJu |
| | | 551 | | |
| | | 552 | | /// <summary> |
| | | 553 | | /// Gets the file path of the bonus question prediction prompt being used by this service |
| | | 554 | | /// </summary> |
| | | 555 | | /// <returns>The absolute file path to the bonus prompt file</returns> |
| | 1 | 556 | | public string GetBonusPromptPath() => _bonusPromptPath; |
| | | 557 | | |
| | | 558 | | /// <summary> |
| | | 559 | | /// Internal class for deserializing the structured prediction response |
| | | 560 | | /// </summary> |
| | | 561 | | private class PredictionResponse |
| | | 562 | | { |
| | | 563 | | [JsonPropertyName("home")] |
| | 1 | 564 | | public int Home { get; set; } |
| | | 565 | | |
| | | 566 | | [JsonPropertyName("away")] |
| | 1 | 567 | | public int Away { get; set; } |
| | | 568 | | |
| | | 569 | | [JsonPropertyName("justification")] |
| | 1 | 570 | | public JustificationResponse? Justification { get; set; } |
| | | 571 | | } |
| | | 572 | | |
| | | 573 | | private class JustificationResponse |
| | | 574 | | { |
| | | 575 | | [JsonPropertyName("keyReasoning")] |
| | 1 | 576 | | public string KeyReasoning { get; set; } = string.Empty; |
| | | 577 | | |
| | | 578 | | [JsonPropertyName("contextSources")] |
| | 1 | 579 | | public JustificationContextSourcesResponse ContextSources { get; set; } = new(); |
| | | 580 | | |
| | | 581 | | [JsonPropertyName("uncertainties")] |
| | 1 | 582 | | public string[] Uncertainties { get; set; } = Array.Empty<string>(); |
| | | 583 | | } |
| | | 584 | | |
| | | 585 | | private class JustificationContextSourcesResponse |
| | | 586 | | { |
| | | 587 | | [JsonPropertyName("mostValuable")] |
| | 1 | 588 | | public JustificationContextSourceEntry[] MostValuable { get; set; } = Array.Empty<JustificationContextSourceEntr |
| | | 589 | | |
| | | 590 | | [JsonPropertyName("leastValuable")] |
| | 1 | 591 | | public JustificationContextSourceEntry[] LeastValuable { get; set; } = Array.Empty<JustificationContextSourceEnt |
| | | 592 | | } |
| | | 593 | | |
| | | 594 | | private class JustificationContextSourceEntry |
| | | 595 | | { |
| | | 596 | | [JsonPropertyName("documentName")] |
| | 1 | 597 | | public string DocumentName { get; set; } = string.Empty; |
| | | 598 | | |
| | | 599 | | [JsonPropertyName("details")] |
| | 1 | 600 | | public string Details { get; set; } = string.Empty; |
| | | 601 | | } |
| | | 602 | | |
| | | 603 | | /// <summary> |
| | | 604 | | /// Internal class for deserializing the bonus predictions response |
| | | 605 | | /// </summary> |
| | | 606 | | private class BonusPredictionsResponse |
| | | 607 | | { |
| | | 608 | | [JsonPropertyName("predictions")] |
| | 0 | 609 | | public BonusPredictionEntry[]? Predictions { get; set; } |
| | | 610 | | } |
| | | 611 | | |
| | | 612 | | /// <summary> |
| | | 613 | | /// Internal class for deserializing individual bonus prediction entries |
| | | 614 | | /// </summary> |
| | | 615 | | private class BonusPredictionEntry |
| | | 616 | | { |
| | | 617 | | [JsonPropertyName("questionId")] |
| | 0 | 618 | | public string QuestionId { get; set; } = string.Empty; |
| | | 619 | | |
| | | 620 | | [JsonPropertyName("selectedOptionIds")] |
| | 0 | 621 | | public string[] SelectedOptionIds { get; set; } = Array.Empty<string>(); |
| | | 622 | | } |
| | | 623 | | |
| | | 624 | | /// <summary> |
| | | 625 | | /// Internal class for deserializing single bonus prediction response |
| | | 626 | | /// </summary> |
| | | 627 | | private class SingleBonusPredictionResponse |
| | | 628 | | { |
| | | 629 | | [JsonPropertyName("selectedOptionIds")] |
| | 1 | 630 | | public string[] SelectedOptionIds { get; set; } = Array.Empty<string>(); |
| | | 631 | | } |
| | | 632 | | |
| | | 633 | | /// <summary> |
| | | 634 | | /// Sets Langfuse-mapped OpenTelemetry attributes on the given activity. |
| | | 635 | | /// If <paramref name="activity"/> is <c>null</c> (no OTel listener registered), this is a no-op. |
| | | 636 | | /// </summary> |
| | | 637 | | private void SetLangfuseGenerationAttributes( |
| | | 638 | | Activity? activity, |
| | | 639 | | List<ChatMessage> messages, |
| | | 640 | | string responseJson, |
| | | 641 | | ChatTokenUsage usage, |
| | | 642 | | PredictionTelemetryMetadata? telemetryMetadata) |
| | | 643 | | { |
| | 1 | 644 | | if (activity is null) |
| | 1 | 645 | | return; |
| | | 646 | | |
| | 0 | 647 | | activity.SetTag("langfuse.observation.type", "generation"); |
| | 0 | 648 | | activity.SetTag("gen_ai.request.model", _model); |
| | | 649 | | |
| | | 650 | | // Serialize messages as input (system prompt + user message) |
| | 0 | 651 | | var inputMessages = messages.Select(m => new |
| | 0 | 652 | | { |
| | 0 | 653 | | role = m switch |
| | 0 | 654 | | { |
| | 0 | 655 | | SystemChatMessage => "system", |
| | 0 | 656 | | UserChatMessage => "user", |
| | 0 | 657 | | _ => "unknown" |
| | 0 | 658 | | }, |
| | 0 | 659 | | content = m switch |
| | 0 | 660 | | { |
| | 0 | 661 | | SystemChatMessage s => s.Content[0].Text, |
| | 0 | 662 | | UserChatMessage u => u.Content[0].Text, |
| | 0 | 663 | | _ => string.Empty |
| | 0 | 664 | | } |
| | 0 | 665 | | }); |
| | 0 | 666 | | activity.SetTag("langfuse.observation.input", JsonSerializer.Serialize(inputMessages)); |
| | 0 | 667 | | activity.SetTag("langfuse.observation.output", responseJson); |
| | 0 | 668 | | telemetryMetadata?.ApplyToObservation(activity); |
| | | 669 | | |
| | | 670 | | // Token usage details |
| | 0 | 671 | | var usageDetails = new |
| | 0 | 672 | | { |
| | 0 | 673 | | input = usage.InputTokenCount, |
| | 0 | 674 | | output = usage.OutputTokenCount, |
| | 0 | 675 | | cache_read_input_tokens = usage.InputTokenDetails?.CachedTokenCount ?? 0, |
| | 0 | 676 | | reasoning_tokens = usage.OutputTokenDetails?.ReasoningTokenCount ?? 0 |
| | 0 | 677 | | }; |
| | 0 | 678 | | activity.SetTag("langfuse.observation.usage_details", JsonSerializer.Serialize(usageDetails)); |
| | | 679 | | |
| | | 680 | | // Cost details are intentionally omitted here: Langfuse automatically infers costs from the |
| | | 681 | | // model name and usage_details using its maintained pricing tables, which are more up-to-date |
| | | 682 | | // than the prices kept in this repository. Explicitly ingesting cost_details would override |
| | | 683 | | // that inference (ingested values take priority over inferred ones). |
| | 0 | 684 | | } |
| | | 685 | | } |