| | | 1 | | using Microsoft.Extensions.Logging; |
| | | 2 | | using Spectre.Console.Cli; |
| | | 3 | | using Spectre.Console; |
| | | 4 | | using System.Globalization; |
| | | 5 | | using System.Text.Json; |
| | | 6 | | using System.Text.Json.Serialization; |
| | | 7 | | using EHonda.KicktippAi.Core; |
| | | 8 | | using Orchestrator.Infrastructure.Factories; |
| | | 9 | | |
| | | 10 | | namespace Orchestrator.Commands.Observability.Cost; |
| | | 11 | | |
| | | 12 | | public class CostCommand : AsyncCommand<CostSettings> |
| | | 13 | | { |
| | | 14 | | private const int FirestoreInFilterLimit = 30; |
| | | 15 | | |
| | 1 | 16 | | private static readonly JsonSerializerOptions OutputJsonOptions = new(JsonSerializerDefaults.Web) |
| | 1 | 17 | | { |
| | 1 | 18 | | WriteIndented = true, |
| | 1 | 19 | | DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull |
| | 1 | 20 | | }; |
| | | 21 | | |
| | | 22 | | private readonly IAnsiConsole _console; |
| | | 23 | | private readonly IFirebaseServiceFactory _firebaseServiceFactory; |
| | | 24 | | private readonly ILogger<CostCommand> _logger; |
| | | 25 | | |
| | 1 | 26 | | public CostCommand( |
| | 1 | 27 | | IAnsiConsole console, |
| | 1 | 28 | | IFirebaseServiceFactory firebaseServiceFactory, |
| | 1 | 29 | | ILogger<CostCommand> logger) |
| | | 30 | | { |
| | 1 | 31 | | _console = console; |
| | 1 | 32 | | _firebaseServiceFactory = firebaseServiceFactory; |
| | 1 | 33 | | _logger = logger; |
| | 1 | 34 | | } |
| | | 35 | | |
| | | 36 | | protected override async Task<int> ExecuteAsync(CommandContext context, CostSettings settings, CancellationToken can |
| | | 37 | | { |
| | | 38 | | |
| | | 39 | | try |
| | | 40 | | { |
| | | 41 | | // Load configuration from file if specified |
| | 1 | 42 | | if (!string.IsNullOrWhiteSpace(settings.ConfigFile)) |
| | | 43 | | { |
| | 1 | 44 | | var fileConfig = await LoadConfigurationFromFile(settings.ConfigFile); |
| | 1 | 45 | | settings = MergeConfigurations(fileConfig, settings); |
| | | 46 | | } |
| | | 47 | | |
| | | 48 | | // Create Firebase services using factory (factory handles env var loading) |
| | 1 | 49 | | var predictionRepository = _firebaseServiceFactory.CreatePredictionRepository(); |
| | | 50 | | |
| | 1 | 51 | | _console.MarkupLine($"[green]Cost command initialized[/]"); |
| | | 52 | | |
| | 1 | 53 | | if (settings.Verbose) |
| | | 54 | | { |
| | 1 | 55 | | _console.MarkupLine("[dim]Verbose mode enabled[/]"); |
| | | 56 | | } |
| | | 57 | | |
| | 1 | 58 | | if (settings.All) |
| | | 59 | | { |
| | 1 | 60 | | _console.MarkupLine("[blue]All mode enabled - aggregating over all available data[/]"); |
| | | 61 | | } |
| | | 62 | | |
| | | 63 | | // Parse filter parameters |
| | 1 | 64 | | var matchdays = ParseMatchdays(settings); |
| | 1 | 65 | | var models = ParseModels(settings); |
| | 1 | 66 | | var reasoningEfforts = ParseReasoningEfforts(settings); |
| | 1 | 67 | | var communityContexts = ParseCommunityContexts(settings); |
| | | 68 | | |
| | | 69 | | // Get available model configurations and community contexts if not specified |
| | 1 | 70 | | var availableModelConfigs = await ResolveModelConfigsAsync(predictionRepository, models, reasoningEfforts, c |
| | 1 | 71 | | var availableCommunityContexts = communityContexts ?? await predictionRepository.GetAvailableCommunityContex |
| | 1 | 72 | | var queryMatchdays = matchdays; |
| | | 73 | | |
| | 1 | 74 | | if (settings.Verbose) |
| | | 75 | | { |
| | 1 | 76 | | _console.MarkupLine($"[dim]Filters:[/]"); |
| | 1 | 77 | | _console.MarkupLine($"[dim] Matchdays: {FormatMatchdayFilter(queryMatchdays)}[/]"); |
| | 1 | 78 | | _console.MarkupLine($"[dim] Model Configs: {(models?.Any() == true || reasoningEfforts?.Any() == true ? |
| | 1 | 79 | | _console.MarkupLine($"[dim] Community Contexts: {(communityContexts?.Any() == true ? string.Join(", ", |
| | 1 | 80 | | _console.MarkupLine($"[dim] Include Bonus: {settings.Bonus || settings.All}[/]"); |
| | | 81 | | } |
| | | 82 | | |
| | | 83 | | // Calculate costs |
| | 1 | 84 | | var totalCost = 0.0; |
| | 1 | 85 | | var matchPredictionCost = 0.0; |
| | 1 | 86 | | var bonusPredictionCost = 0.0; |
| | 1 | 87 | | var matchPredictionCount = 0; |
| | 1 | 88 | | var bonusPredictionCount = 0; |
| | | 89 | | |
| | | 90 | | // Structure to store detailed breakdown data with reprediction index support |
| | 1 | 91 | | var costRows = new List<CostReportRow>(); |
| | | 92 | | |
| | 1 | 93 | | await _console.Status() |
| | 1 | 94 | | .Spinner(Spinner.Known.Dots) |
| | 1 | 95 | | .StartAsync("Calculating costs...", async ctx => |
| | 1 | 96 | | { |
| | 1 | 97 | | foreach (var modelConfig in availableModelConfigs) |
| | 1 | 98 | | { |
| | 1 | 99 | | foreach (var communityContext in availableCommunityContexts) |
| | 1 | 100 | | { |
| | 1 | 101 | | ctx.Status($"Processing {modelConfig.DisplayName} - {communityContext}..."); |
| | 1 | 102 | | |
| | 1 | 103 | | if (settings.Verbose) |
| | 1 | 104 | | { |
| | 1 | 105 | | _console.MarkupLine($"[dim] Processing model config: {modelConfig.DisplayName}, communi |
| | 1 | 106 | | } |
| | 1 | 107 | | |
| | 1 | 108 | | // Get match prediction costs by reprediction index |
| | 1 | 109 | | var matchCostsByIndex = await GetMatchCostsByRepredictionIndexAsync( |
| | 1 | 110 | | predictionRepository, |
| | 1 | 111 | | modelConfig, |
| | 1 | 112 | | communityContext, |
| | 1 | 113 | | queryMatchdays, |
| | 1 | 114 | | cancellationToken); |
| | 1 | 115 | | |
| | 1 | 116 | | foreach (var kvp in matchCostsByIndex) |
| | 1 | 117 | | { |
| | 1 | 118 | | var repredictionIndex = kvp.Key; |
| | 1 | 119 | | var (cost, count) = kvp.Value; |
| | 1 | 120 | | |
| | 1 | 121 | | matchPredictionCost += cost; |
| | 1 | 122 | | matchPredictionCount += count; |
| | 1 | 123 | | |
| | 1 | 124 | | // Store detailed data for breakdown |
| | 1 | 125 | | if (cost > 0 || count > 0) |
| | 1 | 126 | | { |
| | 1 | 127 | | costRows.Add(new CostReportRow(communityContext, modelConfig.Model, modelConfig.Reas |
| | 1 | 128 | | } |
| | 1 | 129 | | |
| | 1 | 130 | | if (settings.Verbose && (cost > 0 || count > 0)) |
| | 1 | 131 | | { |
| | 1 | 132 | | _console.MarkupLine($"[dim] Match predictions (reprediction {repredictionIndex}): |
| | 1 | 133 | | } |
| | 1 | 134 | | } |
| | 1 | 135 | | |
| | 1 | 136 | | // Get bonus prediction costs if requested or if all mode is enabled |
| | 1 | 137 | | if (settings.Bonus || settings.All) |
| | 1 | 138 | | { |
| | 1 | 139 | | var bonusCostsByIndex = await predictionRepository.GetBonusPredictionCostsByReprediction |
| | 1 | 140 | | modelConfig, |
| | 1 | 141 | | communityContext, |
| | 1 | 142 | | cancellationToken); |
| | 1 | 143 | | |
| | 1 | 144 | | foreach (var kvp in bonusCostsByIndex) |
| | 1 | 145 | | { |
| | 1 | 146 | | var repredictionIndex = kvp.Key; |
| | 1 | 147 | | var (cost, count) = kvp.Value; |
| | 1 | 148 | | |
| | 1 | 149 | | bonusPredictionCost += cost; |
| | 1 | 150 | | bonusPredictionCount += count; |
| | 1 | 151 | | |
| | 1 | 152 | | // Store detailed data for breakdown |
| | 1 | 153 | | if (cost > 0 || count > 0) |
| | 1 | 154 | | { |
| | 1 | 155 | | costRows.Add(new CostReportRow(communityContext, modelConfig.Model, modelConfig. |
| | 1 | 156 | | } |
| | 1 | 157 | | |
| | 1 | 158 | | if (settings.Verbose && (cost > 0 || count > 0)) |
| | 1 | 159 | | { |
| | 1 | 160 | | _console.MarkupLine($"[dim] Bonus predictions (reprediction {repredictionInde |
| | 1 | 161 | | } |
| | 1 | 162 | | } |
| | 1 | 163 | | } |
| | 1 | 164 | | } |
| | 1 | 165 | | } |
| | 1 | 166 | | }); |
| | | 167 | | |
| | 1 | 168 | | totalCost = matchPredictionCost + bonusPredictionCost; |
| | | 169 | | |
| | | 170 | | // Display results |
| | 1 | 171 | | var table = new Table(); |
| | 1 | 172 | | table.Border(TableBorder.Rounded); |
| | | 173 | | |
| | 1 | 174 | | if (settings.DetailedBreakdown) |
| | | 175 | | { |
| | | 176 | | // Add columns for detailed breakdown with reprediction support |
| | 1 | 177 | | table.AddColumn("Community Context"); |
| | 1 | 178 | | table.AddColumn("Model"); |
| | 1 | 179 | | table.AddColumn("Category"); |
| | 1 | 180 | | table.AddColumn("Index 0", col => col.RightAligned()); |
| | 1 | 181 | | table.AddColumn("Index 1", col => col.RightAligned()); |
| | 1 | 182 | | table.AddColumn("Index 2+", col => col.RightAligned()); |
| | 1 | 183 | | table.AddColumn("Total Count", col => col.RightAligned()); |
| | 1 | 184 | | table.AddColumn("Total Cost (USD)", col => col.RightAligned()); |
| | | 185 | | |
| | | 186 | | // Group data by community context, model, and category to aggregate reprediction indices |
| | 1 | 187 | | var groupedData = costRows |
| | 1 | 188 | | .GroupBy(d => new { d.CommunityContext, d.Model, d.ReasoningEffort, d.Category }) |
| | 1 | 189 | | .Select(g => new |
| | 1 | 190 | | { |
| | 1 | 191 | | g.Key.CommunityContext, |
| | 1 | 192 | | g.Key.Model, |
| | 1 | 193 | | ReasoningEffort = g.Key.ReasoningEffort ?? "model-default", |
| | 1 | 194 | | ModelConfigDisplayName = PredictionModelConfig.Create(g.Key.Model, g.Key.ReasoningEffort).Displa |
| | 1 | 195 | | g.Key.Category, |
| | 1 | 196 | | Index0Count = g.Where(x => x.RepredictionIndex == 0).Sum(x => x.Count), |
| | 1 | 197 | | Index0Cost = g.Where(x => x.RepredictionIndex == 0).Sum(x => x.Cost), |
| | 1 | 198 | | Index1Count = g.Where(x => x.RepredictionIndex == 1).Sum(x => x.Count), |
| | 1 | 199 | | Index1Cost = g.Where(x => x.RepredictionIndex == 1).Sum(x => x.Cost), |
| | 1 | 200 | | Index2PlusCount = g.Where(x => x.RepredictionIndex >= 2).Sum(x => x.Count), |
| | 1 | 201 | | Index2PlusCost = g.Where(x => x.RepredictionIndex >= 2).Sum(x => x.Cost), |
| | 1 | 202 | | TotalCount = g.Sum(x => x.Count), |
| | 1 | 203 | | TotalCost = g.Sum(x => x.Cost) |
| | 1 | 204 | | }) |
| | 1 | 205 | | .OrderBy(g => g.CommunityContext) |
| | 1 | 206 | | .ThenBy(g => g.Model) |
| | 1 | 207 | | .ThenBy(g => g.ReasoningEffort) |
| | 1 | 208 | | .ThenBy(g => g.Category) |
| | 1 | 209 | | .ToList(); |
| | | 210 | | |
| | | 211 | | // Add rows for detailed breakdown with alternating styling |
| | 1 | 212 | | for (int i = 0; i < groupedData.Count; i++) |
| | | 213 | | { |
| | 1 | 214 | | var data = groupedData[i]; |
| | 1 | 215 | | var isEvenRow = i % 2 == 0; |
| | | 216 | | |
| | 1 | 217 | | var index0Text = data.Index0Count > 0 ? $"{data.Index0Count} (${data.Index0Cost.ToString("F2", Cultu |
| | 1 | 218 | | var index1Text = data.Index1Count > 0 ? $"{data.Index1Count} (${data.Index1Cost.ToString("F2", Cultu |
| | 1 | 219 | | var index2PlusText = data.Index2PlusCount > 0 ? $"{data.Index2PlusCount} (${data.Index2PlusCost.ToSt |
| | | 220 | | |
| | 1 | 221 | | if (isEvenRow) |
| | | 222 | | { |
| | | 223 | | // Even rows - normal styling |
| | 1 | 224 | | table.AddRow( |
| | 1 | 225 | | data.CommunityContext, |
| | 1 | 226 | | data.ModelConfigDisplayName, |
| | 1 | 227 | | data.Category, |
| | 1 | 228 | | index0Text, |
| | 1 | 229 | | index1Text, |
| | 1 | 230 | | index2PlusText, |
| | 1 | 231 | | data.TotalCount.ToString(CultureInfo.InvariantCulture), |
| | 1 | 232 | | $"${data.TotalCost.ToString("F4", CultureInfo.InvariantCulture)}" |
| | 1 | 233 | | ); |
| | | 234 | | } |
| | | 235 | | else |
| | | 236 | | { |
| | | 237 | | // Odd rows - subtle blue tint for visual differentiation |
| | 1 | 238 | | table.AddRow( |
| | 1 | 239 | | $"[blue]{data.CommunityContext}[/]", |
| | 1 | 240 | | $"[blue]{data.ModelConfigDisplayName}[/]", |
| | 1 | 241 | | $"[blue]{data.Category}[/]", |
| | 1 | 242 | | $"[blue]{index0Text}[/]", |
| | 1 | 243 | | $"[blue]{index1Text}[/]", |
| | 1 | 244 | | $"[blue]{index2PlusText}[/]", |
| | 1 | 245 | | $"[blue]{data.TotalCount.ToString(CultureInfo.InvariantCulture)}[/]", |
| | 1 | 246 | | $"[blue]${data.TotalCost.ToString("F4", CultureInfo.InvariantCulture)}[/]" |
| | 1 | 247 | | ); |
| | | 248 | | } |
| | | 249 | | } |
| | | 250 | | |
| | | 251 | | // Add total row |
| | 1 | 252 | | if (costRows.Any()) |
| | | 253 | | { |
| | | 254 | | // Calculate totals by reprediction index |
| | 1 | 255 | | var totalIndex0Count = costRows.Where(x => x.RepredictionIndex == 0).Sum(x => x.Count); |
| | 1 | 256 | | var totalIndex0Cost = costRows.Where(x => x.RepredictionIndex == 0).Sum(x => x.Cost); |
| | 1 | 257 | | var totalIndex1Count = costRows.Where(x => x.RepredictionIndex == 1).Sum(x => x.Count); |
| | 1 | 258 | | var totalIndex1Cost = costRows.Where(x => x.RepredictionIndex == 1).Sum(x => x.Cost); |
| | 1 | 259 | | var totalIndex2PlusCount = costRows.Where(x => x.RepredictionIndex >= 2).Sum(x => x.Count); |
| | 1 | 260 | | var totalIndex2PlusCost = costRows.Where(x => x.RepredictionIndex >= 2).Sum(x => x.Cost); |
| | | 261 | | |
| | 1 | 262 | | var totalIndex0Text = totalIndex0Count > 0 ? $"{totalIndex0Count} (${totalIndex0Cost.ToString("F2", |
| | 1 | 263 | | var totalIndex1Text = totalIndex1Count > 0 ? $"{totalIndex1Count} (${totalIndex1Cost.ToString("F2", |
| | 1 | 264 | | var totalIndex2PlusText = totalIndex2PlusCount > 0 ? $"{totalIndex2PlusCount} (${totalIndex2PlusCost |
| | | 265 | | |
| | 1 | 266 | | table.AddEmptyRow(); |
| | 1 | 267 | | table.AddRow( |
| | 1 | 268 | | "[bold]Total[/]", |
| | 1 | 269 | | "", |
| | 1 | 270 | | "", |
| | 1 | 271 | | $"[bold]{totalIndex0Text}[/]", |
| | 1 | 272 | | $"[bold]{totalIndex1Text}[/]", |
| | 1 | 273 | | $"[bold]{totalIndex2PlusText}[/]", |
| | 1 | 274 | | $"[bold]{(matchPredictionCount + bonusPredictionCount).ToString(CultureInfo.InvariantCulture)}[/ |
| | 1 | 275 | | $"[bold]${totalCost.ToString("F4", CultureInfo.InvariantCulture)}[/]" |
| | 1 | 276 | | ); |
| | | 277 | | } |
| | | 278 | | } |
| | | 279 | | else |
| | | 280 | | { |
| | | 281 | | // Standard summary table |
| | 1 | 282 | | table.AddColumn("Category"); |
| | 1 | 283 | | table.AddColumn("Count", col => col.RightAligned()); |
| | 1 | 284 | | table.AddColumn("Cost (USD)", col => col.RightAligned()); |
| | | 285 | | |
| | 1 | 286 | | var rowIndex = 0; |
| | | 287 | | |
| | | 288 | | // Add Match row |
| | 1 | 289 | | var isEvenRow = rowIndex % 2 == 0; |
| | 1 | 290 | | if (isEvenRow) |
| | | 291 | | { |
| | 1 | 292 | | table.AddRow("Match", matchPredictionCount.ToString(CultureInfo.InvariantCulture), $"${matchPredicti |
| | | 293 | | } |
| | | 294 | | else |
| | | 295 | | { |
| | 0 | 296 | | table.AddRow( |
| | 0 | 297 | | "[blue]Match[/]", |
| | 0 | 298 | | $"[blue]{matchPredictionCount.ToString(CultureInfo.InvariantCulture)}[/]", |
| | 0 | 299 | | $"[blue]${matchPredictionCost.ToString("F4", CultureInfo.InvariantCulture)}[/]" |
| | 0 | 300 | | ); |
| | | 301 | | } |
| | 1 | 302 | | rowIndex++; |
| | | 303 | | |
| | | 304 | | // Add Bonus row if applicable |
| | 1 | 305 | | if (settings.Bonus || settings.All) |
| | | 306 | | { |
| | 1 | 307 | | isEvenRow = rowIndex % 2 == 0; |
| | 1 | 308 | | if (isEvenRow) |
| | | 309 | | { |
| | 0 | 310 | | table.AddRow("Bonus", bonusPredictionCount.ToString(CultureInfo.InvariantCulture), $"${bonusPred |
| | | 311 | | } |
| | | 312 | | else |
| | | 313 | | { |
| | 1 | 314 | | table.AddRow( |
| | 1 | 315 | | "[blue]Bonus[/]", |
| | 1 | 316 | | $"[blue]{bonusPredictionCount.ToString(CultureInfo.InvariantCulture)}[/]", |
| | 1 | 317 | | $"[blue]${bonusPredictionCost.ToString("F4", CultureInfo.InvariantCulture)}[/]" |
| | 1 | 318 | | ); |
| | | 319 | | } |
| | | 320 | | } |
| | | 321 | | |
| | 1 | 322 | | table.AddEmptyRow(); |
| | 1 | 323 | | table.AddRow("[bold]Total[/]", $"[bold]{(matchPredictionCount + bonusPredictionCount).ToString(CultureIn |
| | | 324 | | } |
| | | 325 | | |
| | 1 | 326 | | _console.Write(table); |
| | | 327 | | |
| | 1 | 328 | | if (!string.IsNullOrWhiteSpace(settings.OutputJson)) |
| | | 329 | | { |
| | 1 | 330 | | var report = CreateReport( |
| | 1 | 331 | | settings, |
| | 1 | 332 | | queryMatchdays, |
| | 1 | 333 | | models, |
| | 1 | 334 | | reasoningEfforts, |
| | 1 | 335 | | communityContexts, |
| | 1 | 336 | | costRows, |
| | 1 | 337 | | matchPredictionCount, |
| | 1 | 338 | | matchPredictionCost, |
| | 1 | 339 | | bonusPredictionCount, |
| | 1 | 340 | | bonusPredictionCost, |
| | 1 | 341 | | totalCost); |
| | 1 | 342 | | await WriteJsonReportAsync(settings.OutputJson, report, cancellationToken); |
| | 1 | 343 | | _console.MarkupLine($"[green]✓ Cost JSON written to[/] [yellow]{Path.GetFullPath(settings.OutputJson)}[/ |
| | | 344 | | } |
| | | 345 | | |
| | 1 | 346 | | _console.MarkupLine($"[green]✓ Cost calculation completed[/]"); |
| | | 347 | | |
| | 1 | 348 | | return 0; |
| | | 349 | | } |
| | 1 | 350 | | catch (Exception ex) |
| | | 351 | | { |
| | 1 | 352 | | _logger.LogError(ex, "Failed to calculate costs"); |
| | 1 | 353 | | _console.MarkupLine($"[red]✗ Failed to calculate costs: {ex.Message}[/]"); |
| | 1 | 354 | | return 1; |
| | | 355 | | } |
| | 1 | 356 | | } |
| | | 357 | | |
| | | 358 | | private List<int>? ParseMatchdays(CostSettings settings) |
| | | 359 | | { |
| | 1 | 360 | | if (settings.All || string.IsNullOrWhiteSpace(settings.Matchdays)) |
| | 1 | 361 | | return null; // null means all matchdays |
| | | 362 | | |
| | 1 | 363 | | if (settings.Matchdays.Trim().ToLowerInvariant() == "all") |
| | 1 | 364 | | return null; |
| | | 365 | | |
| | | 366 | | try |
| | | 367 | | { |
| | 1 | 368 | | return settings.Matchdays |
| | 1 | 369 | | .Split(',', StringSplitOptions.RemoveEmptyEntries) |
| | 1 | 370 | | .Select(md => int.Parse(md.Trim())) |
| | 1 | 371 | | .ToList(); |
| | | 372 | | } |
| | 1 | 373 | | catch (FormatException) |
| | | 374 | | { |
| | 1 | 375 | | throw new ArgumentException($"Invalid matchday format: {settings.Matchdays}. Use comma-separated numbers (e. |
| | | 376 | | } |
| | 1 | 377 | | } |
| | | 378 | | |
| | | 379 | | private static string FormatMatchdayFilter(List<int>? matchdays) |
| | | 380 | | { |
| | 1 | 381 | | if (matchdays is null) |
| | | 382 | | { |
| | 1 | 383 | | return "all (unfiltered; no matchday discovery)"; |
| | | 384 | | } |
| | | 385 | | |
| | 1 | 386 | | return matchdays.Count == 0 |
| | 1 | 387 | | ? "none" |
| | 1 | 388 | | : string.Join(", ", matchdays); |
| | | 389 | | } |
| | | 390 | | |
| | | 391 | | private static async Task<Dictionary<int, (double cost, int count)>> GetMatchCostsByRepredictionIndexAsync( |
| | | 392 | | IPredictionRepository predictionRepository, |
| | | 393 | | PredictionModelConfig modelConfig, |
| | | 394 | | string communityContext, |
| | | 395 | | List<int>? matchdays, |
| | | 396 | | CancellationToken cancellationToken) |
| | | 397 | | { |
| | 1 | 398 | | if (matchdays is { Count: 0 }) |
| | | 399 | | { |
| | 0 | 400 | | return []; |
| | | 401 | | } |
| | | 402 | | |
| | 1 | 403 | | if (matchdays is null || matchdays.Count <= FirestoreInFilterLimit) |
| | | 404 | | { |
| | 1 | 405 | | return await predictionRepository.GetMatchPredictionCostsByRepredictionIndexAsync( |
| | 1 | 406 | | modelConfig, |
| | 1 | 407 | | communityContext, |
| | 1 | 408 | | matchdays, |
| | 1 | 409 | | cancellationToken); |
| | | 410 | | } |
| | | 411 | | |
| | 1 | 412 | | var aggregate = new Dictionary<int, (double cost, int count)>(); |
| | | 413 | | |
| | 1 | 414 | | foreach (var chunk in matchdays.Chunk(FirestoreInFilterLimit)) |
| | | 415 | | { |
| | 1 | 416 | | var chunkCosts = await predictionRepository.GetMatchPredictionCostsByRepredictionIndexAsync( |
| | 1 | 417 | | modelConfig, |
| | 1 | 418 | | communityContext, |
| | 1 | 419 | | chunk.ToList(), |
| | 1 | 420 | | cancellationToken); |
| | | 421 | | |
| | 1 | 422 | | foreach (var kvp in chunkCosts) |
| | | 423 | | { |
| | 1 | 424 | | var (existingCost, existingCount) = aggregate.GetValueOrDefault(kvp.Key); |
| | 1 | 425 | | var (chunkCost, chunkCount) = kvp.Value; |
| | 1 | 426 | | aggregate[kvp.Key] = (existingCost + chunkCost, existingCount + chunkCount); |
| | | 427 | | } |
| | | 428 | | } |
| | | 429 | | |
| | 1 | 430 | | return aggregate; |
| | 1 | 431 | | } |
| | | 432 | | |
| | | 433 | | private static CostReport CreateReport( |
| | | 434 | | CostSettings settings, |
| | | 435 | | List<int>? matchdays, |
| | | 436 | | List<string>? models, |
| | | 437 | | List<string?>? reasoningEfforts, |
| | | 438 | | List<string>? communityContexts, |
| | | 439 | | IReadOnlyList<CostReportRow> rows, |
| | | 440 | | int matchPredictionCount, |
| | | 441 | | double matchPredictionCost, |
| | | 442 | | int bonusPredictionCount, |
| | | 443 | | double bonusPredictionCost, |
| | | 444 | | double totalCost) |
| | | 445 | | { |
| | 1 | 446 | | var categoryTotals = rows |
| | 1 | 447 | | .GroupBy(row => new { row.Category, row.RepredictionIndex }) |
| | 1 | 448 | | .Select(group => new CostReportCategoryTotal( |
| | 1 | 449 | | group.Key.Category, |
| | 1 | 450 | | group.Key.RepredictionIndex, |
| | 1 | 451 | | group.Sum(row => row.Count), |
| | 1 | 452 | | group.Sum(row => row.Cost))) |
| | 1 | 453 | | .OrderBy(total => total.Category, StringComparer.Ordinal) |
| | 1 | 454 | | .ThenBy(total => total.RepredictionIndex) |
| | 1 | 455 | | .ToList(); |
| | | 456 | | |
| | 1 | 457 | | return new CostReport( |
| | 1 | 458 | | new CostReportFilters( |
| | 1 | 459 | | matchdays, |
| | 1 | 460 | | models, |
| | 1 | 461 | | reasoningEfforts, |
| | 1 | 462 | | communityContexts, |
| | 1 | 463 | | settings.Bonus || settings.All, |
| | 1 | 464 | | settings.All), |
| | 1 | 465 | | rows |
| | 1 | 466 | | .OrderBy(row => row.CommunityContext, StringComparer.Ordinal) |
| | 1 | 467 | | .ThenBy(row => row.Model, StringComparer.Ordinal) |
| | 1 | 468 | | .ThenBy(row => row.ReasoningEffort ?? string.Empty, StringComparer.Ordinal) |
| | 1 | 469 | | .ThenBy(row => row.Category, StringComparer.Ordinal) |
| | 1 | 470 | | .ThenBy(row => row.RepredictionIndex) |
| | 1 | 471 | | .ToList(), |
| | 1 | 472 | | categoryTotals, |
| | 1 | 473 | | new CostReportTotal( |
| | 1 | 474 | | matchPredictionCount, |
| | 1 | 475 | | matchPredictionCost, |
| | 1 | 476 | | bonusPredictionCount, |
| | 1 | 477 | | bonusPredictionCost, |
| | 1 | 478 | | matchPredictionCount + bonusPredictionCount, |
| | 1 | 479 | | totalCost)); |
| | | 480 | | } |
| | | 481 | | |
| | | 482 | | private static async Task WriteJsonReportAsync( |
| | | 483 | | string outputPath, |
| | | 484 | | CostReport report, |
| | | 485 | | CancellationToken cancellationToken) |
| | | 486 | | { |
| | 1 | 487 | | var resolvedPath = Path.GetFullPath(outputPath); |
| | 1 | 488 | | var directory = Path.GetDirectoryName(resolvedPath); |
| | | 489 | | |
| | 1 | 490 | | if (!string.IsNullOrWhiteSpace(directory)) |
| | | 491 | | { |
| | 1 | 492 | | Directory.CreateDirectory(directory); |
| | | 493 | | } |
| | | 494 | | |
| | 1 | 495 | | await File.WriteAllTextAsync( |
| | 1 | 496 | | resolvedPath, |
| | 1 | 497 | | JsonSerializer.Serialize(report, OutputJsonOptions), |
| | 1 | 498 | | cancellationToken); |
| | 1 | 499 | | } |
| | | 500 | | |
| | | 501 | | private List<string>? ParseModels(CostSettings settings) |
| | | 502 | | { |
| | 1 | 503 | | if (settings.All || string.IsNullOrWhiteSpace(settings.Models)) |
| | 1 | 504 | | return null; // null means all models |
| | | 505 | | |
| | 1 | 506 | | if (settings.Models.Trim().ToLowerInvariant() == "all") |
| | 1 | 507 | | return null; |
| | | 508 | | |
| | 1 | 509 | | return settings.Models |
| | 1 | 510 | | .Split(',', StringSplitOptions.RemoveEmptyEntries) |
| | 1 | 511 | | .Select(m => m.Trim()) |
| | 1 | 512 | | .Where(m => !string.IsNullOrWhiteSpace(m)) |
| | 1 | 513 | | .ToList(); |
| | | 514 | | } |
| | | 515 | | |
| | | 516 | | private static List<string?>? ParseReasoningEfforts(CostSettings settings) |
| | | 517 | | { |
| | 1 | 518 | | if (settings.All || string.IsNullOrWhiteSpace(settings.ReasoningEfforts)) |
| | | 519 | | { |
| | 1 | 520 | | return null; |
| | | 521 | | } |
| | | 522 | | |
| | 1 | 523 | | if (settings.ReasoningEfforts.Trim().Equals("all", StringComparison.OrdinalIgnoreCase)) |
| | | 524 | | { |
| | 0 | 525 | | return null; |
| | | 526 | | } |
| | | 527 | | |
| | 1 | 528 | | var efforts = new List<string?>(); |
| | 1 | 529 | | foreach (var segment in settings.ReasoningEfforts.Split(',', StringSplitOptions.RemoveEmptyEntries)) |
| | | 530 | | { |
| | 1 | 531 | | var trimmed = segment.Trim(); |
| | 1 | 532 | | if (trimmed.Equals("model-default", StringComparison.OrdinalIgnoreCase)) |
| | | 533 | | { |
| | 0 | 534 | | efforts.Add(null); |
| | 0 | 535 | | continue; |
| | | 536 | | } |
| | | 537 | | |
| | 1 | 538 | | if (!PredictionModelConfig.IsValidReasoningEffort(trimmed)) |
| | | 539 | | { |
| | 0 | 540 | | throw new ArgumentException("--reasoning-efforts must contain only: model-default, none, minimal, low, m |
| | | 541 | | } |
| | | 542 | | |
| | 1 | 543 | | efforts.Add(PredictionModelConfig.NormalizeReasoningEffort(trimmed)); |
| | | 544 | | } |
| | | 545 | | |
| | 1 | 546 | | return efforts.Distinct().ToList(); |
| | | 547 | | } |
| | | 548 | | |
| | | 549 | | private static async Task<List<PredictionModelConfig>> ResolveModelConfigsAsync( |
| | | 550 | | IPredictionRepository predictionRepository, |
| | | 551 | | List<string>? models, |
| | | 552 | | List<string?>? reasoningEfforts, |
| | | 553 | | CancellationToken cancellationToken) |
| | | 554 | | { |
| | 1 | 555 | | var availableModelConfigs = await predictionRepository.GetAvailableModelConfigsAsync(cancellationToken); |
| | | 556 | | |
| | 1 | 557 | | var filtered = availableModelConfigs.AsEnumerable(); |
| | 1 | 558 | | if (models is not null) |
| | | 559 | | { |
| | 1 | 560 | | var requestedModels = models.ToHashSet(StringComparer.Ordinal); |
| | 1 | 561 | | filtered = filtered.Where(config => requestedModels.Contains(config.Model)); |
| | | 562 | | } |
| | | 563 | | |
| | 1 | 564 | | if (reasoningEfforts is not null) |
| | | 565 | | { |
| | 1 | 566 | | var requestedEfforts = reasoningEfforts.ToHashSet(StringComparer.Ordinal); |
| | 1 | 567 | | filtered = filtered.Where(config => requestedEfforts.Contains(config.ReasoningEffort)); |
| | | 568 | | } |
| | | 569 | | |
| | 1 | 570 | | return filtered |
| | 1 | 571 | | .OrderBy(config => config.Model, StringComparer.Ordinal) |
| | 1 | 572 | | .ThenBy(config => config.ReasoningEffort ?? string.Empty, StringComparer.Ordinal) |
| | 1 | 573 | | .ToList(); |
| | 1 | 574 | | } |
| | | 575 | | |
| | | 576 | | private List<string>? ParseCommunityContexts(CostSettings settings) |
| | | 577 | | { |
| | 1 | 578 | | if (settings.All || string.IsNullOrWhiteSpace(settings.CommunityContexts)) |
| | 1 | 579 | | return null; // null means all community contexts |
| | | 580 | | |
| | 1 | 581 | | if (settings.CommunityContexts.Trim().ToLowerInvariant() == "all") |
| | 1 | 582 | | return null; |
| | | 583 | | |
| | 1 | 584 | | return settings.CommunityContexts |
| | 1 | 585 | | .Split(',', StringSplitOptions.RemoveEmptyEntries) |
| | 1 | 586 | | .Select(cc => cc.Trim()) |
| | 1 | 587 | | .Where(cc => !string.IsNullOrWhiteSpace(cc)) |
| | 1 | 588 | | .ToList(); |
| | | 589 | | } |
| | | 590 | | |
| | | 591 | | private async Task<CostConfiguration> LoadConfigurationFromFile(string configFilePath) |
| | | 592 | | { |
| | | 593 | | try |
| | | 594 | | { |
| | | 595 | | // Resolve relative paths |
| | 1 | 596 | | var resolvedPath = Path.IsPathRooted(configFilePath) |
| | 1 | 597 | | ? configFilePath |
| | 1 | 598 | | : Path.Combine(Directory.GetCurrentDirectory(), configFilePath); |
| | | 599 | | |
| | 1 | 600 | | if (!File.Exists(resolvedPath)) |
| | | 601 | | { |
| | 1 | 602 | | throw new FileNotFoundException($"Configuration file not found: {resolvedPath}"); |
| | | 603 | | } |
| | | 604 | | |
| | 1 | 605 | | _logger.LogInformation("Loading configuration from: {ConfigPath}", resolvedPath); |
| | | 606 | | |
| | 1 | 607 | | var jsonContent = await File.ReadAllTextAsync(resolvedPath); |
| | 1 | 608 | | var options = new JsonSerializerOptions |
| | 1 | 609 | | { |
| | 1 | 610 | | PropertyNameCaseInsensitive = true, |
| | 1 | 611 | | AllowTrailingCommas = true, |
| | 1 | 612 | | ReadCommentHandling = JsonCommentHandling.Skip |
| | 1 | 613 | | }; |
| | | 614 | | |
| | 1 | 615 | | var config = JsonSerializer.Deserialize<CostConfiguration>(jsonContent, options); |
| | 1 | 616 | | if (config == null) |
| | | 617 | | { |
| | 0 | 618 | | throw new InvalidOperationException($"Failed to deserialize configuration from: {resolvedPath}"); |
| | | 619 | | } |
| | | 620 | | |
| | 1 | 621 | | return config; |
| | | 622 | | } |
| | 1 | 623 | | catch (JsonException ex) |
| | | 624 | | { |
| | 1 | 625 | | throw new InvalidOperationException($"Invalid JSON in configuration file: {configFilePath}. {ex.Message}", e |
| | | 626 | | } |
| | 1 | 627 | | catch (Exception ex) |
| | | 628 | | { |
| | 1 | 629 | | throw new InvalidOperationException($"Failed to load configuration from file: {configFilePath}. {ex.Message} |
| | | 630 | | } |
| | 1 | 631 | | } |
| | | 632 | | |
| | | 633 | | private CostSettings MergeConfigurations(CostConfiguration fileConfig, CostSettings cliSettings) |
| | | 634 | | { |
| | 1 | 635 | | _logger.LogInformation("Merging file configuration with command line options (CLI options take precedence)"); |
| | | 636 | | |
| | | 637 | | // Create a new settings object with file config as base, CLI overrides |
| | 1 | 638 | | var mergedSettings = new CostSettings(); |
| | | 639 | | |
| | | 640 | | // Apply file config first (if values are not null/default) |
| | 1 | 641 | | if (!string.IsNullOrWhiteSpace(fileConfig.Matchdays)) |
| | 1 | 642 | | mergedSettings.Matchdays = fileConfig.Matchdays; |
| | | 643 | | |
| | 1 | 644 | | if (fileConfig.Bonus.HasValue) |
| | 1 | 645 | | mergedSettings.Bonus = fileConfig.Bonus.Value; |
| | | 646 | | |
| | 1 | 647 | | if (!string.IsNullOrWhiteSpace(fileConfig.Models)) |
| | 1 | 648 | | mergedSettings.Models = fileConfig.Models; |
| | | 649 | | |
| | 1 | 650 | | if (!string.IsNullOrWhiteSpace(fileConfig.ReasoningEfforts)) |
| | 0 | 651 | | mergedSettings.ReasoningEfforts = fileConfig.ReasoningEfforts; |
| | | 652 | | |
| | 1 | 653 | | if (!string.IsNullOrWhiteSpace(fileConfig.CommunityContexts)) |
| | 1 | 654 | | mergedSettings.CommunityContexts = fileConfig.CommunityContexts; |
| | | 655 | | |
| | 1 | 656 | | if (fileConfig.All.HasValue) |
| | 1 | 657 | | mergedSettings.All = fileConfig.All.Value; |
| | | 658 | | |
| | 1 | 659 | | if (fileConfig.Verbose.HasValue) |
| | 1 | 660 | | mergedSettings.Verbose = fileConfig.Verbose.Value; |
| | | 661 | | |
| | 1 | 662 | | if (fileConfig.DetailedBreakdown.HasValue) |
| | 1 | 663 | | mergedSettings.DetailedBreakdown = fileConfig.DetailedBreakdown.Value; |
| | | 664 | | |
| | 1 | 665 | | if (!string.IsNullOrWhiteSpace(fileConfig.OutputJson)) |
| | 1 | 666 | | mergedSettings.OutputJson = fileConfig.OutputJson; |
| | | 667 | | |
| | | 668 | | // Override with CLI settings (non-default values) |
| | 1 | 669 | | if (!string.IsNullOrWhiteSpace(cliSettings.Matchdays)) |
| | | 670 | | { |
| | 1 | 671 | | mergedSettings.Matchdays = cliSettings.Matchdays; |
| | 1 | 672 | | if (mergedSettings.Verbose) |
| | 1 | 673 | | _logger.LogInformation("CLI override: Matchdays = {Value}", cliSettings.Matchdays); |
| | | 674 | | } |
| | | 675 | | |
| | 1 | 676 | | if (cliSettings.Bonus) // Only override if explicitly set to true |
| | | 677 | | { |
| | 1 | 678 | | mergedSettings.Bonus = cliSettings.Bonus; |
| | 1 | 679 | | if (mergedSettings.Verbose) |
| | 1 | 680 | | _logger.LogInformation("CLI override: Bonus = {Value}", cliSettings.Bonus); |
| | | 681 | | } |
| | | 682 | | |
| | 1 | 683 | | if (!string.IsNullOrWhiteSpace(cliSettings.Models)) |
| | | 684 | | { |
| | 1 | 685 | | mergedSettings.Models = cliSettings.Models; |
| | 1 | 686 | | if (mergedSettings.Verbose) |
| | 1 | 687 | | _logger.LogInformation("CLI override: Models = {Value}", cliSettings.Models); |
| | | 688 | | } |
| | | 689 | | |
| | 1 | 690 | | if (!string.IsNullOrWhiteSpace(cliSettings.ReasoningEfforts)) |
| | | 691 | | { |
| | 0 | 692 | | mergedSettings.ReasoningEfforts = cliSettings.ReasoningEfforts; |
| | 0 | 693 | | if (mergedSettings.Verbose) |
| | 0 | 694 | | _logger.LogInformation("CLI override: ReasoningEfforts = {Value}", cliSettings.ReasoningEfforts); |
| | | 695 | | } |
| | | 696 | | |
| | 1 | 697 | | if (!string.IsNullOrWhiteSpace(cliSettings.CommunityContexts)) |
| | | 698 | | { |
| | 1 | 699 | | mergedSettings.CommunityContexts = cliSettings.CommunityContexts; |
| | 1 | 700 | | if (mergedSettings.Verbose) |
| | 1 | 701 | | _logger.LogInformation("CLI override: CommunityContexts = {Value}", cliSettings.CommunityContexts); |
| | | 702 | | } |
| | | 703 | | |
| | 1 | 704 | | if (cliSettings.All) // Only override if explicitly set to true |
| | | 705 | | { |
| | 1 | 706 | | mergedSettings.All = cliSettings.All; |
| | 1 | 707 | | if (mergedSettings.Verbose) |
| | 1 | 708 | | _logger.LogInformation("CLI override: All = {Value}", cliSettings.All); |
| | | 709 | | } |
| | | 710 | | |
| | 1 | 711 | | if (cliSettings.Verbose) // Only override if explicitly set to true |
| | | 712 | | { |
| | 1 | 713 | | mergedSettings.Verbose = cliSettings.Verbose; |
| | | 714 | | } |
| | | 715 | | |
| | 1 | 716 | | if (cliSettings.DetailedBreakdown) // Only override if explicitly set to true |
| | | 717 | | { |
| | 1 | 718 | | mergedSettings.DetailedBreakdown = cliSettings.DetailedBreakdown; |
| | 1 | 719 | | if (mergedSettings.Verbose) |
| | 1 | 720 | | _logger.LogInformation("CLI override: DetailedBreakdown = {Value}", cliSettings.DetailedBreakdown); |
| | | 721 | | } |
| | | 722 | | |
| | 1 | 723 | | if (!string.IsNullOrWhiteSpace(cliSettings.OutputJson)) |
| | | 724 | | { |
| | 0 | 725 | | mergedSettings.OutputJson = cliSettings.OutputJson; |
| | 0 | 726 | | if (mergedSettings.Verbose) |
| | 0 | 727 | | _logger.LogInformation("CLI override: OutputJson = {Value}", cliSettings.OutputJson); |
| | | 728 | | } |
| | | 729 | | |
| | | 730 | | // Always preserve the ConfigFile setting |
| | 1 | 731 | | mergedSettings.ConfigFile = cliSettings.ConfigFile; |
| | | 732 | | |
| | 1 | 733 | | return mergedSettings; |
| | | 734 | | } |
| | | 735 | | |
| | 1 | 736 | | private sealed record CostReport( |
| | 1 | 737 | | CostReportFilters Filters, |
| | 1 | 738 | | IReadOnlyList<CostReportRow> Rows, |
| | 1 | 739 | | IReadOnlyList<CostReportCategoryTotal> CategoryTotals, |
| | 1 | 740 | | CostReportTotal Total); |
| | | 741 | | |
| | 1 | 742 | | private sealed record CostReportFilters( |
| | 1 | 743 | | IReadOnlyList<int>? Matchdays, |
| | 1 | 744 | | IReadOnlyList<string>? Models, |
| | 1 | 745 | | IReadOnlyList<string?>? ReasoningEfforts, |
| | 1 | 746 | | IReadOnlyList<string>? CommunityContexts, |
| | 1 | 747 | | bool IncludeBonus, |
| | 1 | 748 | | bool All); |
| | | 749 | | |
| | 1 | 750 | | private sealed record CostReportRow( |
| | 1 | 751 | | string CommunityContext, |
| | 1 | 752 | | string Model, |
| | 1 | 753 | | string? ReasoningEffort, |
| | 1 | 754 | | string ModelConfigKey, |
| | 1 | 755 | | string ModelConfigDisplayName, |
| | 1 | 756 | | string Category, |
| | 1 | 757 | | int RepredictionIndex, |
| | 1 | 758 | | int Count, |
| | 1 | 759 | | double Cost); |
| | | 760 | | |
| | 1 | 761 | | private sealed record CostReportCategoryTotal( |
| | 1 | 762 | | string Category, |
| | 1 | 763 | | int RepredictionIndex, |
| | 1 | 764 | | int Count, |
| | 1 | 765 | | double Cost); |
| | | 766 | | |
| | 1 | 767 | | private sealed record CostReportTotal( |
| | 1 | 768 | | int MatchPredictionCount, |
| | 1 | 769 | | double MatchPredictionCost, |
| | 1 | 770 | | int BonusPredictionCount, |
| | 1 | 771 | | double BonusPredictionCost, |
| | 1 | 772 | | int Count, |
| | 1 | 773 | | double Cost); |
| | | 774 | | } |