< Summary

Information
Class: Orchestrator.Commands.Operations.Verify.VerifyBonusCommand
Assembly: Orchestrator
File(s): /home/runner/work/KicktippAi/KicktippAi/src/Orchestrator/Commands/Operations/Verify/VerifyBonusCommand.cs
Line coverage
99%
Covered lines: 175
Uncovered lines: 1
Coverable lines: 176
Total lines: 393
Line coverage: 99.4%
Branch coverage
95%
Covered branches: 103
Total branches: 108
Branch coverage: 95.3%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Metrics

MethodBranch coverage Crap Score Cyclomatic complexity Line coverage
.ctor(...)100%11100%
ExecuteAsync()100%88100%
ExecuteVerificationWorkflow()96.77%6262100%
ValidateBonusPrediction(...)100%1010100%
CheckBonusPredictionOutdated()100%1616100%
CompareBonusPredictions(...)75%121285.71%

File(s)

/home/runner/work/KicktippAi/KicktippAi/src/Orchestrator/Commands/Operations/Verify/VerifyBonusCommand.cs

#LineLine coverage
 1using EHonda.KicktippAi.Core;
 2using Microsoft.Extensions.Logging;
 3using Spectre.Console.Cli;
 4using Spectre.Console;
 5using KicktippIntegration;
 6using Orchestrator.Commands.Shared;
 7using Orchestrator.Infrastructure;
 8using Orchestrator.Infrastructure.Factories;
 9
 10namespace Orchestrator.Commands.Operations.Verify;
 11
 12public class VerifyBonusCommand : AsyncCommand<VerifySettings>
 13{
 14    private readonly IAnsiConsole _console;
 15    private readonly IFirebaseServiceFactory _firebaseServiceFactory;
 16    private readonly IKicktippClientFactory _kicktippClientFactory;
 17    private readonly ILogger<VerifyBonusCommand> _logger;
 18
 119    public VerifyBonusCommand(
 120        IAnsiConsole console,
 121        IFirebaseServiceFactory firebaseServiceFactory,
 122        IKicktippClientFactory kicktippClientFactory,
 123        ILogger<VerifyBonusCommand> logger)
 24    {
 125        _console = console;
 126        _firebaseServiceFactory = firebaseServiceFactory;
 127        _kicktippClientFactory = kicktippClientFactory;
 128        _logger = logger;
 129    }
 30
 31    protected override async Task<int> ExecuteAsync(CommandContext context, VerifySettings settings, CancellationToken c
 32    {
 33
 34        try
 35        {
 136            _console.MarkupLine($"[green]Verify bonus command initialized[/]");
 37
 138            if (settings.Verbose)
 39            {
 140                _console.MarkupLine("[dim]Verbose mode enabled[/]");
 41            }
 42
 143            if (settings.Agent)
 44            {
 145                _console.MarkupLine("[blue]Agent mode enabled - prediction details will be hidden[/]");
 46            }
 47
 148            if (settings.InitMatchday)
 49            {
 150                _console.MarkupLine("[cyan]Init bonus mode enabled - will return error if no predictions exist[/]");
 51            }
 52
 153            if (settings.CheckOutdated)
 54            {
 155                _console.MarkupLine("[cyan]Outdated check enabled - predictions will be checked against latest context d
 56            }
 57
 58            // Execute the verification workflow
 159            var hasDiscrepancies = await ExecuteVerificationWorkflow(settings);
 60
 161            return hasDiscrepancies ? 1 : 0;
 62        }
 163        catch (Exception ex)
 64        {
 165            _logger.LogError(ex, "Error executing verify bonus command");
 166            _console.MarkupLine($"[red]Error:[/] {ex.Message}");
 167            return 1;
 68        }
 169    }
 70
 71    private async Task<bool> ExecuteVerificationWorkflow(VerifySettings settings)
 72    {
 173        var kicktippClient = _kicktippClientFactory.CreateClient();
 174        string communityContext = settings.CommunityContext ?? settings.Community;
 175        var competition = CompetitionResolver.ResolveCompetition(settings.Competition, settings.Community, communityCont
 176        var modelConfig = PredictionServiceCommandSupport.CreateModelConfig(settings.Model, settings.ReasoningEffort);
 177        var repositoryCompetition = CompetitionResolver.ToRepositoryCompetitionArgument(competition);
 78
 79        // Try to get the prediction repository (may be null if Firebase is not configured)
 180        var predictionRepository = _firebaseServiceFactory.CreatePredictionRepository(repositoryCompetition);
 181        if (predictionRepository == null)
 82        {
 183            _console.MarkupLine("[red]Error: Database not configured. Cannot verify predictions without database access.
 184            _console.MarkupLine("[yellow]Hint: Set FIREBASE_PROJECT_ID and FIREBASE_SERVICE_ACCOUNT_JSON environment var
 185            return true; // Consider this a failure
 86        }
 87
 88        // Get KPI repository for outdated checks (required for bonus predictions)
 189        var kpiRepository = _firebaseServiceFactory.CreateKpiRepository(repositoryCompetition);
 90
 191        _console.MarkupLine($"[blue]Using community:[/] [yellow]{settings.Community}[/]");
 192        _console.MarkupLine($"[blue]Using community context:[/] [yellow]{communityContext}[/]");
 193        _console.MarkupLine($"[blue]Using competition:[/] [yellow]{competition}[/]");
 194        _console.MarkupLine($"[blue]Using model config:[/] [yellow]{modelConfig.DisplayName}[/]");
 195        _console.MarkupLine("[blue]Getting open bonus questions from Kicktipp...[/]");
 96
 97        // Step 1: Get open bonus questions from Kicktipp
 198        var bonusQuestions = await kicktippClient.GetOpenBonusQuestionsAsync(settings.Community);
 99
 1100        if (!bonusQuestions.Any())
 101        {
 1102            _console.MarkupLine("[yellow]No bonus questions found on Kicktipp[/]");
 1103            return false;
 104        }
 105
 1106        _console.MarkupLine($"[green]Found {bonusQuestions.Count} bonus questions on Kicktipp[/]");
 107
 1108        _console.MarkupLine("[blue]Getting placed bonus predictions from Kicktipp...[/]");
 109
 110        // Step 1.5: Get currently placed predictions from Kicktipp
 1111        var placedPredictions = await kicktippClient.GetPlacedBonusPredictionsAsync(settings.Community);
 112
 1113        _console.MarkupLine("[blue]Retrieving predictions from database...[/]");
 114
 1115        var hasDiscrepancies = false;
 1116        var totalQuestions = 0;
 1117        var questionsWithDatabasePredictions = 0;
 1118        var validPredictions = 0;
 119
 120        // Step 2: For each bonus question, check if we have a prediction in database
 1121        foreach (var question in bonusQuestions)
 122        {
 1123            totalQuestions++;
 124
 125            try
 126            {
 127                // Get prediction from database
 1128                if (settings.Verbose)
 129                {
 1130                    _console.MarkupLine($"[dim]  Looking up: {Markup.Escape(question.Text)}[/]");
 131                }
 132
 1133                var databasePrediction = await predictionRepository.GetBonusPredictionByTextAsync(question.Text, modelCo
 1134                var kicktippPrediction = placedPredictions.GetValueOrDefault(question.FormFieldName ?? question.Text);
 135
 1136                if (databasePrediction != null)
 137                {
 1138                    questionsWithDatabasePredictions++;
 139
 140                    // Validate the prediction against the question
 1141                    var isValidPrediction = ValidateBonusPrediction(question, databasePrediction);
 142
 143                    // Compare database prediction with Kicktipp placed prediction
 1144                    var predictionsMatch = CompareBonusPredictions(databasePrediction, kicktippPrediction);
 145
 146                    // Check if prediction is outdated (if enabled)
 1147                    var isOutdated = false;
 1148                    if (settings.CheckOutdated)
 149                    {
 1150                        isOutdated = await CheckBonusPredictionOutdated(predictionRepository, kpiRepository, question.Te
 151                    }
 152
 153                    // Consider prediction valid if it passes validation, matches Kicktipp, and is not outdated
 1154                    var isPredictionValid = isValidPrediction && predictionsMatch && !isOutdated;
 155
 1156                    if (isPredictionValid)
 157                    {
 1158                        validPredictions++;
 159
 1160                        if (settings.Verbose)
 161                        {
 1162                            if (settings.Agent)
 163                            {
 1164                                _console.MarkupLine($"[green]✓ {Markup.Escape(question.Text)}[/] [dim](valid)[/]");
 165                            }
 166                            else
 167                            {
 1168                                var optionTexts = question.Options
 1169                                    .Where(o => databasePrediction.SelectedOptionIds.Contains(o.Id))
 1170                                    .Select(o => o.Text);
 1171                                _console.MarkupLine($"[green]✓ {Markup.Escape(question.Text)}:[/] {string.Join(", ", opt
 172                            }
 173                        }
 174                    }
 175                    else
 176                    {
 1177                        hasDiscrepancies = true;
 178
 1179                        if (settings.Agent)
 180                        {
 1181                            var status = !isValidPrediction ? "invalid prediction" :
 1182                                        !predictionsMatch ? "mismatch with Kicktipp" : "outdated";
 1183                            _console.MarkupLine($"[red]✗ {Markup.Escape(question.Text)}[/] [dim]({status})[/]");
 184                        }
 185                        else
 186                        {
 1187                            if (!isValidPrediction)
 188                            {
 1189                                var optionTexts = question.Options
 1190                                    .Where(o => databasePrediction.SelectedOptionIds.Contains(o.Id))
 1191                                    .Select(o => o.Text);
 1192                                _console.MarkupLine($"[red]✗ {Markup.Escape(question.Text)}:[/] {string.Join(", ", optio
 193                            }
 1194                            else if (!predictionsMatch)
 195                            {
 196                                // Show mismatch details
 1197                                var databaseTexts = question.Options
 1198                                    .Where(o => databasePrediction.SelectedOptionIds.Contains(o.Id))
 1199                                    .Select(o => o.Text);
 1200                                var kicktippTexts = kicktippPrediction != null
 1201                                    ? question.Options
 1202                                        .Where(o => kicktippPrediction.SelectedOptionIds.Contains(o.Id))
 1203                                        .Select(o => o.Text)
 1204                                    : new List<string>();
 205
 1206                                _console.MarkupLine($"[red]✗ {Markup.Escape(question.Text)}:[/]");
 1207                                _console.MarkupLine($"  [yellow]Database:[/] {string.Join(", ", databaseTexts)}");
 1208                                _console.MarkupLine($"  [yellow]Kicktipp:[/] {(kicktippTexts.Any() ? string.Join(", ", k
 209                            }
 1210                            else if (isOutdated)
 211                            {
 1212                                var optionTexts = question.Options
 1213                                    .Where(o => databasePrediction.SelectedOptionIds.Contains(o.Id))
 1214                                    .Select(o => o.Text);
 1215                                _console.MarkupLine($"[red]✗ {Markup.Escape(question.Text)}:[/] {string.Join(", ", optio
 1216                                _console.MarkupLine($"  [yellow]Status:[/] Outdated (context updated after prediction)")
 217                            }
 218                        }
 219                    }
 220                }
 221                else
 222                {
 1223                    hasDiscrepancies = true;
 224
 1225                    if (settings.Verbose)
 226                    {
 1227                        if (settings.Agent)
 228                        {
 1229                            _console.MarkupLine($"[yellow]○ {Markup.Escape(question.Text)}[/] [dim](no prediction)[/]");
 230                        }
 231                        else
 232                        {
 1233                            _console.MarkupLine($"[yellow]○ {Markup.Escape(question.Text)}:[/] [dim](no prediction)[/]")
 234                        }
 235                    }
 236                }
 1237            }
 1238            catch (Exception ex)
 239            {
 1240                hasDiscrepancies = true;
 1241                _logger.LogError(ex, "Error verifying bonus prediction for question '{QuestionText}'", question.Text);
 242
 1243                if (settings.Agent)
 244                {
 1245                    _console.MarkupLine($"[red]✗ {Markup.Escape(question.Text)}[/] [dim](error)[/]");
 246                }
 247                else
 248                {
 1249                    _console.MarkupLine($"[red]✗ {Markup.Escape(question.Text)}:[/] Error during verification");
 250                }
 1251            }
 1252        }
 253
 254        // Step 3: Display summary
 1255        _console.WriteLine();
 1256        _console.MarkupLine("[bold]Verification Summary:[/]");
 1257        _console.MarkupLine($"  Total bonus questions: {totalQuestions}");
 1258        _console.MarkupLine($"  Questions with database predictions: {questionsWithDatabasePredictions}");
 1259        _console.MarkupLine($"  Valid predictions: {validPredictions}");
 260
 261        // Check for init-bonus mode first
 1262        if (settings.InitMatchday && questionsWithDatabasePredictions == 0)
 263        {
 1264            _console.MarkupLine("[yellow]  Init bonus detected - no database predictions exist[/]");
 1265            _console.MarkupLine("[red]Returning error to trigger initial prediction workflow[/]");
 1266            return true; // Return error to trigger workflow
 267        }
 268
 1269        if (hasDiscrepancies)
 270        {
 1271            _console.MarkupLine($"[red]  Missing or invalid predictions: {totalQuestions - validPredictions}[/]");
 1272            _console.MarkupLine("[red]Verification failed - some predictions are missing or invalid[/]");
 273        }
 274        else
 275        {
 1276            _console.MarkupLine("[green]  All predictions are valid - verification successful[/]");
 277        }
 278
 1279        return hasDiscrepancies;
 1280    }
 281
 282    private static bool ValidateBonusPrediction(BonusQuestion question, BonusPrediction prediction)
 283    {
 284        // Check if all selected option IDs exist in the question
 1285        var validOptionIds = question.Options.Select(o => o.Id).ToHashSet();
 1286        var allOptionsValid = prediction.SelectedOptionIds.All(id => validOptionIds.Contains(id));
 287
 1288        if (!allOptionsValid)
 289        {
 1290            return false;
 291        }
 292
 293        // Check if the number of selections is valid
 1294        var selectionCount = prediction.SelectedOptionIds.Count;
 1295        if (selectionCount < 1 || selectionCount > question.MaxSelections)
 296        {
 1297            return false;
 298        }
 299
 300        // Check for duplicates
 1301        var uniqueSelections = prediction.SelectedOptionIds.Distinct().Count();
 1302        if (uniqueSelections != selectionCount)
 303        {
 1304            return false;
 305        }
 306
 1307        return true;
 308    }
 309
 310    private async Task<bool> CheckBonusPredictionOutdated(
 311        IPredictionRepository predictionRepository,
 312        IKpiRepository kpiRepository,
 313        string questionText,
 314        PredictionModelConfig modelConfig,
 315        string communityContext,
 316        bool verbose)
 317    {
 318        try
 319        {
 320            // Get prediction metadata (includes creation timestamp and context document names)
 1321            var predictionMetadata = await predictionRepository.GetBonusPredictionMetadataByTextAsync(
 1322                questionText, modelConfig, communityContext);
 323
 1324            if (predictionMetadata == null)
 325            {
 326                // No metadata found, assume not outdated
 1327                return false;
 328            }
 329
 330            // Check if any KPI document has been updated after the prediction was created
 1331            foreach (var contextDocumentName in predictionMetadata.ContextDocumentNames)
 332            {
 1333                var kpiDocument = await kpiRepository.GetKpiDocumentAsync(contextDocumentName, communityContext);
 1334                if (kpiDocument != null)
 335                {
 336                    // Compare the creation timestamps
 337                    // Note: We need to be careful about timezone handling here
 338                    // Both timestamps should be in UTC for proper comparison
 1339                    if (kpiDocument.CreatedAt > predictionMetadata.CreatedAt)
 340                    {
 1341                        if (verbose)
 342                        {
 1343                            _console.MarkupLine($"[yellow]KPI document '{contextDocumentName}' updated after prediction 
 1344                            _console.MarkupLine($"  [dim]Prediction created:[/] {predictionMetadata.CreatedAt:yyyy-MM-dd
 1345                            _console.MarkupLine($"  [dim]KPI document created:[/] {kpiDocument.CreatedAt:yyyy-MM-dd HH:m
 346                        }
 1347                        return true; // Prediction is outdated
 348                    }
 349
 1350                    if (verbose)
 351                    {
 1352                        _console.MarkupLine($"[dim]KPI document '{contextDocumentName}' found, version {kpiDocument.Vers
 353                    }
 354                }
 1355                else if (verbose)
 356                {
 1357                    _console.MarkupLine($"[yellow]Warning: KPI document '{contextDocumentName}' not found[/]");
 358                }
 1359            }
 360
 1361            return false; // No KPI documents are newer than the prediction
 362        }
 1363        catch (Exception ex)
 364        {
 1365            if (verbose)
 366            {
 1367                _console.MarkupLine($"[yellow]Warning: Could not check if prediction is outdated: {ex.Message}[/]");
 368            }
 1369            return false; // Assume not outdated if we can't determine
 370        }
 1371    }
 372
 373    private static bool CompareBonusPredictions(BonusPrediction? databasePrediction, BonusPrediction? kicktippPrediction
 374    {
 375        // Both null - match
 1376        if (databasePrediction == null && kicktippPrediction == null)
 377        {
 0378            return true;
 379        }
 380
 381        // One null, other not - mismatch
 1382        if (databasePrediction == null || kicktippPrediction == null)
 383        {
 1384            return false;
 385        }
 386
 387        // Both have values - compare selected option IDs
 1388        var databaseOptions = databasePrediction.SelectedOptionIds.OrderBy(x => x).ToList();
 1389        var kicktippOptions = kicktippPrediction.SelectedOptionIds.OrderBy(x => x).ToList();
 390
 1391        return databaseOptions.SequenceEqual(kicktippOptions);
 392    }
 393}
Generated by: ReportGenerator 5.5.10.0
06/15/2026 - 23:01:29
GitHub | reportgenerator.io

Methods/Properties

.ctor(Spectre.Console.IAnsiConsole, Orchestrator.Infrastructure.Factories.IFirebaseServiceFactory, Orchestrator.Infrastructure.Factories.IKicktippClientFactory, Microsoft.Extensions.Logging.ILogger<Orchestrator.Commands.Operations.Verify.VerifyBonusCommand>)
ExecuteAsync()
ExecuteVerificationWorkflow()
ValidateBonusPrediction(EHonda.KicktippAi.Core.BonusQuestion, EHonda.KicktippAi.Core.BonusPrediction)
CheckBonusPredictionOutdated()
CompareBonusPredictions(EHonda.KicktippAi.Core.BonusPrediction, EHonda.KicktippAi.Core.BonusPrediction)