| | | 1 | | using System.Text.Json; |
| | | 2 | | using EHonda.KicktippAi.Core; |
| | | 3 | | using Microsoft.Extensions.Logging; |
| | | 4 | | using Spectre.Console.Cli; |
| | | 5 | | using Spectre.Console; |
| | | 6 | | using OpenAiIntegration; |
| | | 7 | | using Orchestrator.Commands.Operations.Matchday; |
| | | 8 | | using Orchestrator.Commands.Shared; |
| | | 9 | | using Orchestrator.Infrastructure; |
| | | 10 | | using Orchestrator.Infrastructure.Factories; |
| | | 11 | | using Orchestrator.Infrastructure.Langfuse; |
| | | 12 | | |
| | | 13 | | namespace Orchestrator.Commands.Operations.Bonus; |
| | | 14 | | |
| | | 15 | | public class BonusCommand : AsyncCommand<BaseSettings> |
| | | 16 | | { |
| | | 17 | | private const string FifaRankingsDocumentName = "fifa-rankings"; |
| | | 18 | | |
| | | 19 | | private readonly IAnsiConsole _console; |
| | | 20 | | private readonly IFirebaseServiceFactory _firebaseServiceFactory; |
| | | 21 | | private readonly IKicktippClientFactory _kicktippClientFactory; |
| | | 22 | | private readonly IOpenAiServiceFactory _openAiServiceFactory; |
| | | 23 | | private readonly IContextProviderFactory _contextProviderFactory; |
| | | 24 | | private readonly ILogger<BonusCommand> _logger; |
| | | 25 | | private readonly ILangfusePublicApiClient? _langfuseClient; |
| | | 26 | | |
| | 1 | 27 | | public BonusCommand( |
| | 1 | 28 | | IAnsiConsole console, |
| | 1 | 29 | | IFirebaseServiceFactory firebaseServiceFactory, |
| | 1 | 30 | | IKicktippClientFactory kicktippClientFactory, |
| | 1 | 31 | | IOpenAiServiceFactory openAiServiceFactory, |
| | 1 | 32 | | IContextProviderFactory contextProviderFactory, |
| | 1 | 33 | | ILogger<BonusCommand> logger, |
| | 1 | 34 | | ILangfusePublicApiClient? langfuseClient = null) |
| | | 35 | | { |
| | 1 | 36 | | _console = console; |
| | 1 | 37 | | _firebaseServiceFactory = firebaseServiceFactory; |
| | 1 | 38 | | _kicktippClientFactory = kicktippClientFactory; |
| | 1 | 39 | | _openAiServiceFactory = openAiServiceFactory; |
| | 1 | 40 | | _contextProviderFactory = contextProviderFactory; |
| | 1 | 41 | | _logger = logger; |
| | 1 | 42 | | _langfuseClient = langfuseClient; |
| | 1 | 43 | | } |
| | | 44 | | |
| | | 45 | | protected override async Task<int> ExecuteAsync(CommandContext context, BaseSettings settings, CancellationToken can |
| | | 46 | | { |
| | 1 | 47 | | return await ExecuteWithSettingsAsync(settings, cancellationToken); |
| | 1 | 48 | | } |
| | | 49 | | |
| | | 50 | | internal async Task<int> ExecuteWithSettingsAsync(BaseSettings settings, CancellationToken cancellationToken = defau |
| | | 51 | | { |
| | | 52 | | |
| | | 53 | | try |
| | | 54 | | { |
| | 1 | 55 | | var initialModel = string.IsNullOrWhiteSpace(settings.Model) ? "(competition default)" : settings.Model; |
| | 1 | 56 | | _console.MarkupLine($"[green]Bonus command initialized with model:[/] [yellow]{initialModel}[/]"); |
| | | 57 | | |
| | 1 | 58 | | if (settings.Verbose) |
| | | 59 | | { |
| | 1 | 60 | | _console.MarkupLine("[dim]Verbose mode enabled[/]"); |
| | | 61 | | } |
| | | 62 | | |
| | 1 | 63 | | if (settings.OverrideKicktipp) |
| | | 64 | | { |
| | 1 | 65 | | _console.MarkupLine("[yellow]Override mode enabled - will override existing Kicktipp predictions[/]"); |
| | | 66 | | } |
| | | 67 | | |
| | 1 | 68 | | if (settings.OverrideDatabase) |
| | | 69 | | { |
| | 1 | 70 | | _console.MarkupLine("[yellow]Override database mode enabled - will override existing database prediction |
| | | 71 | | } |
| | | 72 | | |
| | 1 | 73 | | if (settings.Agent) |
| | | 74 | | { |
| | 1 | 75 | | _console.MarkupLine("[blue]Agent mode enabled - prediction details will be hidden[/]"); |
| | | 76 | | } |
| | | 77 | | |
| | 1 | 78 | | if (settings.DryRun) |
| | | 79 | | { |
| | 1 | 80 | | _console.MarkupLine("[magenta]Dry run mode enabled - no changes will be made to database or Kicktipp[/]" |
| | | 81 | | } |
| | | 82 | | |
| | 1 | 83 | | if (!string.IsNullOrEmpty(settings.EstimatedCostsModel)) |
| | | 84 | | { |
| | 1 | 85 | | _console.MarkupLine($"[cyan]Estimated costs will be calculated for model:[/] [yellow]{settings.Estimated |
| | | 86 | | } |
| | | 87 | | |
| | | 88 | | // Validate reprediction settings |
| | 1 | 89 | | if (settings.OverrideDatabase && settings.IsRepredictMode) |
| | | 90 | | { |
| | 1 | 91 | | _console.MarkupLine($"[red]Error:[/] --override-database cannot be used with reprediction flags (--repre |
| | 1 | 92 | | return 1; |
| | | 93 | | } |
| | | 94 | | |
| | 1 | 95 | | if (settings.MaxRepredictions.HasValue && settings.MaxRepredictions.Value < 0) |
| | | 96 | | { |
| | 1 | 97 | | _console.MarkupLine($"[red]Error:[/] --max-repredictions must be 0 or greater"); |
| | 1 | 98 | | return 1; |
| | | 99 | | } |
| | | 100 | | |
| | 1 | 101 | | if (settings.IsRepredictMode) |
| | | 102 | | { |
| | 1 | 103 | | var maxValue = settings.MaxRepredictions ?? int.MaxValue; |
| | 1 | 104 | | _console.MarkupLine($"[yellow]Reprediction mode enabled - max repredictions: {(settings.MaxRepredictions |
| | | 105 | | } |
| | | 106 | | |
| | | 107 | | // Execute the bonus prediction workflow |
| | 1 | 108 | | await ExecuteBonusWorkflow(settings); |
| | | 109 | | |
| | 1 | 110 | | return 0; |
| | | 111 | | } |
| | 1 | 112 | | catch (Exception ex) |
| | | 113 | | { |
| | 1 | 114 | | _logger.LogError(ex, "Error executing bonus command"); |
| | 1 | 115 | | _console.MarkupLine($"[red]Error:[/] {ex.Message}"); |
| | 1 | 116 | | return 1; |
| | | 117 | | } |
| | 1 | 118 | | } |
| | | 119 | | |
| | | 120 | | /// <summary> |
| | | 121 | | /// Communities that have production workflows invoking the bonus command. |
| | | 122 | | /// Update this set when adding or removing community bonus workflows in .github/workflows/. |
| | | 123 | | /// See .github/workflows/AGENTS.md for details. |
| | | 124 | | /// </summary> |
| | 1 | 125 | | private static readonly HashSet<string> ProductionCommunities = new(StringComparer.OrdinalIgnoreCase) |
| | 1 | 126 | | { |
| | 1 | 127 | | "schadensfresse", |
| | 1 | 128 | | "pes-squad", |
| | 1 | 129 | | "ehonda-ai-arena", |
| | 1 | 130 | | "rabetrabauken2026" |
| | 1 | 131 | | }; |
| | | 132 | | |
| | | 133 | | private async Task ExecuteBonusWorkflow(BaseSettings settings) |
| | | 134 | | { |
| | | 135 | | // Start root OTel activity for Langfuse trace |
| | 1 | 136 | | using var activity = Telemetry.Source.StartActivity("bonus"); |
| | | 137 | | |
| | | 138 | | // Set Langfuse environment based on community |
| | 1 | 139 | | var environment = ProductionCommunities.Contains(settings.Community) ? "production" : "development"; |
| | 1 | 140 | | LangfuseActivityPropagation.SetEnvironment(activity, environment); |
| | | 141 | | |
| | 1 | 142 | | string communityContext = settings.CommunityContext ?? settings.Community; |
| | 1 | 143 | | var competition = CompetitionResolver.ResolveCompetition(settings.Competition, settings.Community, communityCont |
| | 1 | 144 | | var modelConfig = PredictionServiceCommandSupport.CreateModelConfig(settings.Model, settings.ReasoningEffort); |
| | 1 | 145 | | var model = modelConfig.Model; |
| | 1 | 146 | | var repositoryCompetition = CompetitionResolver.ToRepositoryCompetitionArgument(competition); |
| | | 147 | | |
| | | 148 | | // Set Langfuse trace-level attributes |
| | 1 | 149 | | var sessionId = $"bonus-{settings.Community}"; |
| | 1 | 150 | | var traceTags = new[] { settings.Community, model, competition }; |
| | 1 | 151 | | LangfuseActivityPropagation.SetSessionId(activity, sessionId); |
| | 1 | 152 | | LangfuseActivityPropagation.SetTraceTags(activity, traceTags); |
| | 1 | 153 | | LangfuseActivityPropagation.SetTraceMetadata(activity, "community", settings.Community); |
| | 1 | 154 | | LangfuseActivityPropagation.SetTraceMetadata(activity, "competition", competition); |
| | 1 | 155 | | LangfuseActivityPropagation.SetTraceMetadata(activity, "model", model); |
| | 1 | 156 | | if (modelConfig.ReasoningEffort is not null) |
| | | 157 | | { |
| | 1 | 158 | | LangfuseActivityPropagation.SetTraceMetadata(activity, "reasoningEffort", modelConfig.ReasoningEffort); |
| | | 159 | | } |
| | 1 | 160 | | LangfuseActivityPropagation.SetTraceMetadata(activity, "repredictMode", settings.IsRepredictMode ? "true" : "fal |
| | | 161 | | |
| | | 162 | | // Note: trace input is set after bonus questions are fetched |
| | | 163 | | |
| | | 164 | | // Create services using factories |
| | 1 | 165 | | var kicktippClient = _kicktippClientFactory.CreateClient(); |
| | 1 | 166 | | var predictionService = PredictionServiceCommandSupport.CreatePredictionService( |
| | 1 | 167 | | _openAiServiceFactory, |
| | 1 | 168 | | _langfuseClient, |
| | 1 | 169 | | _console, |
| | 1 | 170 | | model, |
| | 1 | 171 | | competition, |
| | 1 | 172 | | settings.Community, |
| | 1 | 173 | | communityContext, |
| | 1 | 174 | | settings.PromptSource, |
| | 1 | 175 | | settings.LangfusePromptName, |
| | 1 | 176 | | settings.LangfusePromptLabel, |
| | 1 | 177 | | settings.LangfusePromptVersion, |
| | 1 | 178 | | modelConfig.ReasoningEffort, |
| | 1 | 179 | | settings.MaxOutputTokenCount, |
| | 1 | 180 | | bonusPrompt: true); |
| | | 181 | | |
| | | 182 | | // Log the prompt paths being used |
| | 1 | 183 | | if (settings.Verbose) |
| | | 184 | | { |
| | 1 | 185 | | _console.MarkupLine($"[dim]Bonus prompt:[/] [blue]{predictionService.GetBonusPromptPath()}[/]"); |
| | | 186 | | } |
| | | 187 | | |
| | | 188 | | // Create KPI Context Provider for bonus predictions using factory |
| | 1 | 189 | | var kpiContextProvider = _contextProviderFactory.CreateKpiContextProvider(repositoryCompetition); |
| | 1 | 190 | | if (CompetitionResolver.IsWorldCupCompetition(competition)) |
| | | 191 | | { |
| | 1 | 192 | | await EnsureWorldCupRankingKpiPresentAsync(kpiContextProvider, communityContext); |
| | | 193 | | } |
| | | 194 | | |
| | 1 | 195 | | var tokenUsageTracker = _openAiServiceFactory.GetTokenUsageTracker(); |
| | | 196 | | |
| | | 197 | | // Create repositories |
| | 1 | 198 | | var predictionRepository = _firebaseServiceFactory.CreatePredictionRepository(repositoryCompetition); |
| | 1 | 199 | | var kpiRepository = _firebaseServiceFactory.CreateKpiRepository(repositoryCompetition); |
| | 1 | 200 | | var databaseEnabled = true; |
| | | 201 | | |
| | | 202 | | // Reset token usage tracker for this workflow |
| | 1 | 203 | | tokenUsageTracker.Reset(); |
| | | 204 | | |
| | 1 | 205 | | LangfuseActivityPropagation.SetTraceMetadata(activity, "communityContext", communityContext); |
| | | 206 | | |
| | 1 | 207 | | _console.MarkupLine($"[blue]Using community:[/] [yellow]{settings.Community}[/]"); |
| | 1 | 208 | | _console.MarkupLine($"[blue]Using community context:[/] [yellow]{communityContext}[/]"); |
| | 1 | 209 | | _console.MarkupLine($"[blue]Using competition:[/] [yellow]{competition}[/]"); |
| | 1 | 210 | | _console.MarkupLine("[blue]Getting open bonus questions from Kicktipp...[/]"); |
| | | 211 | | |
| | | 212 | | // Step 1: Get open bonus questions from Kicktipp |
| | 1 | 213 | | var bonusQuestions = await kicktippClient.GetOpenBonusQuestionsAsync(settings.Community); |
| | | 214 | | |
| | 1 | 215 | | if (!bonusQuestions.Any()) |
| | | 216 | | { |
| | 1 | 217 | | _console.MarkupLine("[yellow]No open bonus questions found[/]"); |
| | 1 | 218 | | return; |
| | | 219 | | } |
| | | 220 | | |
| | 1 | 221 | | _console.MarkupLine($"[green]Found {bonusQuestions.Count} open bonus questions[/]"); |
| | | 222 | | |
| | | 223 | | // Set trace input now that we know the questions |
| | 1 | 224 | | var traceInput = new |
| | 1 | 225 | | { |
| | 1 | 226 | | community = settings.Community, |
| | 1 | 227 | | model, |
| | 1 | 228 | | competition, |
| | 1 | 229 | | questions = bonusQuestions.Select(q => q.Text).ToArray() |
| | 1 | 230 | | }; |
| | 1 | 231 | | activity?.SetTag("langfuse.trace.input", JsonSerializer.Serialize(traceInput)); |
| | | 232 | | |
| | 1 | 233 | | if (databaseEnabled) |
| | | 234 | | { |
| | 1 | 235 | | _console.MarkupLine("[blue]Database enabled - checking for existing predictions...[/]"); |
| | | 236 | | } |
| | | 237 | | |
| | 1 | 238 | | var predictions = new Dictionary<string, BonusPrediction>(); |
| | 1 | 239 | | var traceRepredictionIndices = new HashSet<string>(StringComparer.Ordinal); |
| | | 240 | | |
| | | 241 | | // Step 2: For each question, check database first, then predict if needed |
| | 1 | 242 | | foreach (var question in bonusQuestions) |
| | | 243 | | { |
| | 1 | 244 | | _console.MarkupLine($"[cyan]Processing:[/] {Markup.Escape(question.Text)}"); |
| | | 245 | | |
| | | 246 | | try |
| | | 247 | | { |
| | 1 | 248 | | BonusPrediction? prediction = null; |
| | 1 | 249 | | bool fromDatabase = false; |
| | 1 | 250 | | bool shouldPredict = false; |
| | 1 | 251 | | int? predictionRepredictionIndex = settings.IsRepredictMode ? null : 0; |
| | | 252 | | |
| | | 253 | | // Check if we have an existing prediction in the database |
| | 1 | 254 | | if (databaseEnabled && !settings.OverrideDatabase && !settings.IsRepredictMode) |
| | | 255 | | { |
| | | 256 | | // Look for prediction by question text, model, and community context |
| | 1 | 257 | | prediction = await predictionRepository!.GetBonusPredictionByTextAsync(question.Text, modelConfig, c |
| | 1 | 258 | | if (prediction != null) |
| | | 259 | | { |
| | 1 | 260 | | fromDatabase = true; |
| | 1 | 261 | | if (settings.Agent) |
| | | 262 | | { |
| | 1 | 263 | | _console.MarkupLine($"[green] ✓ Found existing prediction[/] [dim](from database)[/]"); |
| | | 264 | | } |
| | | 265 | | else |
| | | 266 | | { |
| | 1 | 267 | | var optionTexts = question.Options |
| | 1 | 268 | | .Where(o => prediction.SelectedOptionIds.Contains(o.Id)) |
| | 1 | 269 | | .Select(o => o.Text); |
| | 1 | 270 | | _console.MarkupLine($"[green] ✓ Found existing prediction:[/] {string.Join(", ", optionText |
| | | 271 | | } |
| | | 272 | | } |
| | | 273 | | } |
| | | 274 | | |
| | | 275 | | // Handle reprediction logic |
| | 1 | 276 | | if (settings.IsRepredictMode && databaseEnabled) |
| | | 277 | | { |
| | 1 | 278 | | var currentRepredictionIndex = await predictionRepository!.GetBonusRepredictionIndexAsync(question.T |
| | | 279 | | |
| | 1 | 280 | | if (currentRepredictionIndex == -1) |
| | | 281 | | { |
| | | 282 | | // No prediction exists yet - create first prediction |
| | 1 | 283 | | shouldPredict = true; |
| | 1 | 284 | | predictionRepredictionIndex = 0; |
| | 1 | 285 | | _console.MarkupLine($"[yellow] → No existing prediction found, creating first prediction...[/]" |
| | | 286 | | } |
| | | 287 | | else |
| | | 288 | | { |
| | | 289 | | // Check if we can create another reprediction |
| | 1 | 290 | | var maxAllowed = settings.MaxRepredictions ?? int.MaxValue; |
| | 1 | 291 | | var nextIndex = currentRepredictionIndex + 1; |
| | | 292 | | |
| | 1 | 293 | | if (nextIndex <= maxAllowed) |
| | | 294 | | { |
| | 1 | 295 | | var isOutdated = await CheckBonusPredictionOutdated( |
| | 1 | 296 | | predictionRepository!, |
| | 1 | 297 | | kpiRepository, |
| | 1 | 298 | | question.Text, |
| | 1 | 299 | | modelConfig, |
| | 1 | 300 | | communityContext, |
| | 1 | 301 | | settings.Verbose); |
| | | 302 | | |
| | 1 | 303 | | if (isOutdated) |
| | | 304 | | { |
| | 1 | 305 | | shouldPredict = true; |
| | 1 | 306 | | predictionRepredictionIndex = nextIndex; |
| | 1 | 307 | | _console.MarkupLine($"[yellow] → Creating reprediction {nextIndex} (current: {currentRe |
| | | 308 | | } |
| | | 309 | | else |
| | | 310 | | { |
| | 1 | 311 | | traceRepredictionIndices.Add(currentRepredictionIndex.ToString()); |
| | 1 | 312 | | _console.MarkupLine($"[green] ✓ Skipped reprediction - current prediction is up-to-date |
| | | 313 | | |
| | 1 | 314 | | prediction = await predictionRepository!.GetBonusPredictionByTextAsync(question.Text, mo |
| | 1 | 315 | | if (prediction != null) |
| | | 316 | | { |
| | 1 | 317 | | fromDatabase = true; |
| | 1 | 318 | | if (!settings.Agent) |
| | | 319 | | { |
| | 1 | 320 | | var optionTexts = question.Options |
| | 1 | 321 | | .Where(o => prediction.SelectedOptionIds.Contains(o.Id)) |
| | 1 | 322 | | .Select(o => o.Text); |
| | 1 | 323 | | _console.MarkupLine($"[green] ✓ Latest prediction:[/] {string.Join(", ", option |
| | | 324 | | } |
| | | 325 | | } |
| | | 326 | | } |
| | | 327 | | } |
| | | 328 | | else |
| | | 329 | | { |
| | 1 | 330 | | traceRepredictionIndices.Add(currentRepredictionIndex.ToString()); |
| | 1 | 331 | | _console.MarkupLine($"[yellow] ✗ Skipped - already at max repredictions ({currentRepredicti |
| | | 332 | | |
| | | 333 | | // Get the latest prediction for display purposes |
| | 1 | 334 | | prediction = await predictionRepository!.GetBonusPredictionByTextAsync(question.Text, modelC |
| | 1 | 335 | | if (prediction != null) |
| | | 336 | | { |
| | 1 | 337 | | fromDatabase = true; |
| | 1 | 338 | | if (!settings.Agent) |
| | | 339 | | { |
| | 1 | 340 | | var optionTexts = question.Options |
| | 1 | 341 | | .Where(o => prediction.SelectedOptionIds.Contains(o.Id)) |
| | 1 | 342 | | .Select(o => o.Text); |
| | 1 | 343 | | _console.MarkupLine($"[green] ✓ Latest prediction:[/] {string.Join(", ", optionText |
| | | 344 | | } |
| | | 345 | | } |
| | | 346 | | } |
| | | 347 | | } |
| | | 348 | | } |
| | | 349 | | |
| | | 350 | | // If no existing prediction (normal mode) or we need to predict (reprediction mode), generate a new one |
| | 1 | 351 | | if (prediction == null || shouldPredict) |
| | | 352 | | { |
| | 1 | 353 | | _console.MarkupLine($"[yellow] → Generating new prediction...[/]"); |
| | | 354 | | |
| | | 355 | | // Step 3: Get KPI context for bonus predictions |
| | 1 | 356 | | var contextDocuments = new List<DocumentContext>(); |
| | | 357 | | |
| | | 358 | | // Use KPI documents as context for bonus predictions (targeted by question content) |
| | 1 | 359 | | await foreach (var context in kpiContextProvider.GetBonusQuestionContextAsync(question.Text, communi |
| | | 360 | | { |
| | 1 | 361 | | contextDocuments.Add(context); |
| | | 362 | | } |
| | | 363 | | |
| | 1 | 364 | | if (settings.Verbose) |
| | | 365 | | { |
| | 1 | 366 | | _console.MarkupLine($"[dim] Using {contextDocuments.Count} KPI context documents[/]"); |
| | | 367 | | } |
| | | 368 | | |
| | 1 | 369 | | var telemetryMetadata = new PredictionTelemetryMetadata( |
| | 1 | 370 | | RepredictionIndex: predictionRepredictionIndex); |
| | | 371 | | |
| | | 372 | | // Predict the bonus question |
| | 1 | 373 | | prediction = await predictionService.PredictBonusQuestionAsync(question, contextDocuments, telemetry |
| | | 374 | | |
| | 1 | 375 | | if (prediction != null) |
| | | 376 | | { |
| | 1 | 377 | | if (predictionRepredictionIndex.HasValue) |
| | | 378 | | { |
| | 1 | 379 | | traceRepredictionIndices.Add(predictionRepredictionIndex.Value.ToString()); |
| | | 380 | | } |
| | | 381 | | |
| | 1 | 382 | | if (settings.Agent) |
| | | 383 | | { |
| | 1 | 384 | | _console.MarkupLine($"[green] ✓ Generated prediction[/]"); |
| | | 385 | | } |
| | | 386 | | else |
| | | 387 | | { |
| | 1 | 388 | | var optionTexts = question.Options |
| | 1 | 389 | | .Where(o => prediction.SelectedOptionIds.Contains(o.Id)) |
| | 1 | 390 | | .Select(o => o.Text); |
| | 1 | 391 | | _console.MarkupLine($"[green] ✓ Generated prediction:[/] {string.Join(", ", optionTexts)}") |
| | | 392 | | } |
| | | 393 | | |
| | | 394 | | // Save to database immediately if enabled |
| | 1 | 395 | | if (databaseEnabled && !settings.DryRun) |
| | | 396 | | { |
| | | 397 | | try |
| | | 398 | | { |
| | | 399 | | // Get token usage and cost information |
| | 1 | 400 | | var cost = (double)tokenUsageTracker.GetLastCost(); // Get the cost for this individual |
| | | 401 | | // Use the new GetLastUsageJson method to get full JSON |
| | 1 | 402 | | var tokenUsageJson = tokenUsageTracker.GetLastUsageJson() ?? "{}"; |
| | | 403 | | |
| | 1 | 404 | | if (settings.IsRepredictMode) |
| | | 405 | | { |
| | | 406 | | // Save as reprediction with specific index |
| | 1 | 407 | | var currentIndex = await predictionRepository!.GetBonusRepredictionIndexAsync(questi |
| | 1 | 408 | | var nextIndex = currentIndex == -1 ? 0 : currentIndex + 1; |
| | | 409 | | |
| | 1 | 410 | | await predictionRepository!.SaveBonusRepredictionAsync( |
| | 1 | 411 | | question, |
| | 1 | 412 | | prediction, |
| | 1 | 413 | | modelConfig, |
| | 1 | 414 | | tokenUsageJson, |
| | 1 | 415 | | cost, |
| | 1 | 416 | | communityContext, |
| | 1 | 417 | | contextDocuments.Select(d => d.Name), |
| | 1 | 418 | | nextIndex); |
| | | 419 | | |
| | 1 | 420 | | if (settings.Verbose) |
| | | 421 | | { |
| | 1 | 422 | | _console.MarkupLine($"[dim] ✓ Saved as reprediction {nextIndex} to database[/ |
| | | 423 | | } |
| | | 424 | | } |
| | | 425 | | else |
| | | 426 | | { |
| | | 427 | | // Save normally (override or new prediction) |
| | 1 | 428 | | await predictionRepository!.SaveBonusPredictionAsync( |
| | 1 | 429 | | question, |
| | 1 | 430 | | prediction, |
| | 1 | 431 | | modelConfig, |
| | 1 | 432 | | tokenUsageJson, |
| | 1 | 433 | | cost, |
| | 1 | 434 | | communityContext, |
| | 1 | 435 | | contextDocuments.Select(d => d.Name), |
| | 1 | 436 | | overrideCreatedAt: settings.OverrideDatabase); |
| | | 437 | | |
| | 1 | 438 | | if (settings.Verbose) |
| | | 439 | | { |
| | 1 | 440 | | _console.MarkupLine($"[dim] ✓ Saved to database[/]"); |
| | | 441 | | } |
| | | 442 | | } |
| | 1 | 443 | | } |
| | 1 | 444 | | catch (Exception ex) |
| | | 445 | | { |
| | 1 | 446 | | _logger.LogError(ex, "Failed to save bonus prediction for question '{QuestionText}'", qu |
| | 1 | 447 | | _console.MarkupLine($"[red] ✗ Failed to save to database: {ex.Message}[/]"); |
| | 1 | 448 | | } |
| | | 449 | | } |
| | 1 | 450 | | else if (databaseEnabled && settings.DryRun && settings.Verbose) |
| | | 451 | | { |
| | 1 | 452 | | _console.MarkupLine($"[dim] (Dry run - skipped database save)[/]"); |
| | | 453 | | } |
| | | 454 | | |
| | | 455 | | // Show individual question token usage in verbose mode |
| | 1 | 456 | | if (settings.Verbose) |
| | | 457 | | { |
| | 1 | 458 | | var questionUsage = !string.IsNullOrEmpty(settings.EstimatedCostsModel) |
| | 1 | 459 | | ? tokenUsageTracker.GetLastUsageCompactSummaryWithEstimatedCosts(settings.EstimatedCosts |
| | 1 | 460 | | : tokenUsageTracker.GetLastUsageCompactSummary(); |
| | 1 | 461 | | _console.MarkupLine($"[dim] Token usage: {questionUsage}[/]"); |
| | | 462 | | } |
| | | 463 | | } |
| | | 464 | | else |
| | | 465 | | { |
| | 1 | 466 | | _console.MarkupLine($"[red] ✗ Failed to generate prediction[/]"); |
| | 1 | 467 | | continue; |
| | | 468 | | } |
| | 1 | 469 | | } |
| | | 470 | | |
| | 1 | 471 | | predictions[question.FormFieldName ?? question.Text] = prediction; |
| | | 472 | | |
| | 1 | 473 | | if (!fromDatabase && settings.Verbose) |
| | | 474 | | { |
| | 1 | 475 | | _console.MarkupLine($"[dim] Ready for Kicktipp placement[/]"); |
| | | 476 | | } |
| | 1 | 477 | | } |
| | 1 | 478 | | catch (Exception ex) |
| | | 479 | | { |
| | 1 | 480 | | _logger.LogError(ex, "Error processing bonus question '{QuestionText}'", question.Text); |
| | 1 | 481 | | _console.MarkupLine($"[red] ✗ Error processing question: {ex.Message}[/]"); |
| | 1 | 482 | | } |
| | 1 | 483 | | } |
| | | 484 | | |
| | 1 | 485 | | if (traceRepredictionIndices.Count > 0) |
| | | 486 | | { |
| | 1 | 487 | | LangfuseActivityPropagation.SetTraceMetadata(activity, "repredictionIndices", PredictionTelemetryMetadata.Bu |
| | 1 | 488 | | LangfuseActivityPropagation.SetTraceMetadata(activity, "hasRepredictions", traceRepredictionIndices.Any(inde |
| | | 489 | | } |
| | | 490 | | |
| | 1 | 491 | | if (!predictions.Any()) |
| | | 492 | | { |
| | 1 | 493 | | _console.MarkupLine("[yellow]No predictions available, nothing to place[/]"); |
| | 1 | 494 | | activity?.SetTag("langfuse.trace.output", JsonSerializer.Serialize(new { error = "No predictions available" |
| | 1 | 495 | | return; |
| | | 496 | | } |
| | | 497 | | |
| | | 498 | | // Set trace output with all bonus predictions |
| | 1 | 499 | | var traceOutput = predictions.Select(p => new |
| | 1 | 500 | | { |
| | 1 | 501 | | question = p.Key, |
| | 1 | 502 | | selectedOptionIds = p.Value.SelectedOptionIds |
| | 1 | 503 | | }).ToArray(); |
| | 1 | 504 | | activity?.SetTag("langfuse.trace.output", JsonSerializer.Serialize(traceOutput)); |
| | | 505 | | |
| | | 506 | | // Step 4: Place all predictions using PlaceBonusPredictionsAsync |
| | 1 | 507 | | _console.MarkupLine($"[blue]Placing {predictions.Count} bonus predictions to Kicktipp...[/]"); |
| | | 508 | | |
| | 1 | 509 | | if (settings.DryRun) |
| | | 510 | | { |
| | 1 | 511 | | _console.MarkupLine($"[magenta]✓ Dry run mode - would have placed {predictions.Count} bonus predictions (no |
| | | 512 | | } |
| | | 513 | | else |
| | | 514 | | { |
| | 1 | 515 | | var success = await kicktippClient.PlaceBonusPredictionsAsync(settings.Community, predictions, overridePredi |
| | | 516 | | |
| | 1 | 517 | | if (success) |
| | | 518 | | { |
| | 1 | 519 | | _console.MarkupLine($"[green]✓ Successfully placed all {predictions.Count} bonus predictions![/]"); |
| | | 520 | | } |
| | | 521 | | else |
| | | 522 | | { |
| | 1 | 523 | | _console.MarkupLine("[red]✗ Failed to place some or all bonus predictions[/]"); |
| | | 524 | | } |
| | | 525 | | } |
| | | 526 | | |
| | | 527 | | // Display token usage summary |
| | 1 | 528 | | var summary = !string.IsNullOrEmpty(settings.EstimatedCostsModel) |
| | 1 | 529 | | ? tokenUsageTracker.GetCompactSummaryWithEstimatedCosts(settings.EstimatedCostsModel) |
| | 1 | 530 | | : tokenUsageTracker.GetCompactSummary(); |
| | 1 | 531 | | _console.MarkupLine($"[dim]Token usage (uncached/cached/reasoning/output/$cost): {summary}[/]"); |
| | 1 | 532 | | } |
| | | 533 | | |
| | | 534 | | private static async Task EnsureWorldCupRankingKpiPresentAsync( |
| | | 535 | | IKpiContextProvider kpiContextProvider, |
| | | 536 | | string communityContext) |
| | | 537 | | { |
| | 1 | 538 | | await foreach (var context in kpiContextProvider.GetContextAsync(communityContext)) |
| | | 539 | | { |
| | 1 | 540 | | if (string.Equals(context.Name, FifaRankingsDocumentName, StringComparison.OrdinalIgnoreCase)) |
| | | 541 | | { |
| | | 542 | | return; |
| | | 543 | | } |
| | | 544 | | } |
| | | 545 | | |
| | 1 | 546 | | throw new InvalidOperationException( |
| | 1 | 547 | | "Missing required WM26 KPI context document 'fifa-rankings'. " + |
| | 1 | 548 | | "Run collect-context fifa for this community context."); |
| | 1 | 549 | | } |
| | | 550 | | |
| | | 551 | | private async Task<bool> CheckBonusPredictionOutdated( |
| | | 552 | | IPredictionRepository predictionRepository, |
| | | 553 | | IKpiRepository kpiRepository, |
| | | 554 | | string questionText, |
| | | 555 | | PredictionModelConfig modelConfig, |
| | | 556 | | string communityContext, |
| | | 557 | | bool verbose) |
| | | 558 | | { |
| | | 559 | | try |
| | | 560 | | { |
| | 1 | 561 | | var predictionMetadata = await predictionRepository.GetBonusPredictionMetadataByTextAsync( |
| | 1 | 562 | | questionText, modelConfig, communityContext); |
| | | 563 | | |
| | 1 | 564 | | if (predictionMetadata == null) |
| | | 565 | | { |
| | 0 | 566 | | return false; |
| | | 567 | | } |
| | | 568 | | |
| | 1 | 569 | | foreach (var contextDocumentName in predictionMetadata.ContextDocumentNames) |
| | | 570 | | { |
| | 1 | 571 | | var kpiDocument = await kpiRepository.GetKpiDocumentAsync(contextDocumentName, communityContext); |
| | 1 | 572 | | if (kpiDocument != null) |
| | | 573 | | { |
| | 1 | 574 | | if (kpiDocument.CreatedAt > predictionMetadata.CreatedAt) |
| | | 575 | | { |
| | 1 | 576 | | if (verbose) |
| | | 577 | | { |
| | 1 | 578 | | _console.MarkupLine($"[yellow]KPI document '{contextDocumentName}' updated after prediction |
| | 1 | 579 | | _console.MarkupLine($" [dim]Prediction created:[/] {predictionMetadata.CreatedAt:yyyy-MM-dd |
| | 1 | 580 | | _console.MarkupLine($" [dim]KPI document created:[/] {kpiDocument.CreatedAt:yyyy-MM-dd HH:m |
| | | 581 | | } |
| | | 582 | | |
| | 1 | 583 | | return true; |
| | | 584 | | } |
| | | 585 | | |
| | 1 | 586 | | if (verbose) |
| | | 587 | | { |
| | 0 | 588 | | _console.MarkupLine($"[dim]KPI document '{contextDocumentName}' found, version {kpiDocument.Vers |
| | | 589 | | } |
| | | 590 | | } |
| | 0 | 591 | | else if (verbose) |
| | | 592 | | { |
| | 0 | 593 | | _console.MarkupLine($"[yellow]Warning: KPI document '{contextDocumentName}' not found[/]"); |
| | | 594 | | } |
| | 1 | 595 | | } |
| | | 596 | | |
| | 1 | 597 | | return false; |
| | | 598 | | } |
| | 0 | 599 | | catch (Exception ex) |
| | | 600 | | { |
| | 0 | 601 | | if (verbose) |
| | | 602 | | { |
| | 0 | 603 | | _console.MarkupLine($"[yellow]Warning: Could not check if prediction is outdated: {ex.Message}[/]"); |
| | | 604 | | } |
| | | 605 | | |
| | 0 | 606 | | return false; |
| | | 607 | | } |
| | 1 | 608 | | } |
| | | 609 | | } |