| | | 1 | | using System.Globalization; |
| | | 2 | | using System.Text.Json; |
| | | 3 | | using EHonda.KicktippAi.Core; |
| | | 4 | | using Microsoft.Extensions.Logging; |
| | | 5 | | using OpenAiIntegration; |
| | | 6 | | using Orchestrator.Commands.Observability; |
| | | 7 | | using Orchestrator.Infrastructure.Factories; |
| | | 8 | | using Spectre.Console; |
| | | 9 | | using Spectre.Console.Cli; |
| | | 10 | | |
| | | 11 | | namespace Orchestrator.Commands.Observability.ExportExperimentItem; |
| | | 12 | | |
| | | 13 | | public sealed class ExportExperimentItemCommand : AsyncCommand<ExportExperimentItemSettings> |
| | | 14 | | { |
| | 1 | 15 | | private static readonly JsonSerializerOptions OutputJsonOptions = new() |
| | 1 | 16 | | { |
| | 1 | 17 | | PropertyNamingPolicy = JsonNamingPolicy.CamelCase, |
| | 1 | 18 | | WriteIndented = true |
| | 1 | 19 | | }; |
| | | 20 | | |
| | | 21 | | private readonly IAnsiConsole _console; |
| | | 22 | | private readonly IFirebaseServiceFactory _firebaseServiceFactory; |
| | | 23 | | private readonly ILogger<ExportExperimentItemCommand> _logger; |
| | | 24 | | |
| | 1 | 25 | | public ExportExperimentItemCommand( |
| | 1 | 26 | | IAnsiConsole console, |
| | 1 | 27 | | IFirebaseServiceFactory firebaseServiceFactory, |
| | 1 | 28 | | ILogger<ExportExperimentItemCommand> logger) |
| | | 29 | | { |
| | 1 | 30 | | _console = console; |
| | 1 | 31 | | _firebaseServiceFactory = firebaseServiceFactory; |
| | 1 | 32 | | _logger = logger; |
| | 1 | 33 | | } |
| | | 34 | | |
| | | 35 | | protected override async Task<int> ExecuteAsync(CommandContext context, ExportExperimentItemSettings settings, Cance |
| | | 36 | | { |
| | | 37 | | try |
| | | 38 | | { |
| | 1 | 39 | | _console.MarkupLine($"[green]Exporting experiment item for:[/] [yellow]{Markup.Escape(settings.HomeTeam)}[/] |
| | | 40 | | |
| | 1 | 41 | | var predictionRepository = _firebaseServiceFactory.CreatePredictionRepository(); |
| | 1 | 42 | | var contextRepository = _firebaseServiceFactory.CreateContextRepository(); |
| | 1 | 43 | | var matchOutcomeRepository = _firebaseServiceFactory.CreateMatchOutcomeRepository(); |
| | | 44 | | |
| | 1 | 45 | | var reconstructionService = new MatchPromptReconstructionService( |
| | 1 | 46 | | predictionRepository, |
| | 1 | 47 | | contextRepository, |
| | 1 | 48 | | new InstructionsTemplateProvider(PromptsFileProvider.Create())); |
| | | 49 | | |
| | 1 | 50 | | var evaluationTime = EvaluationTimeParser.ParseOrNull(settings.EvaluationTime); |
| | 1 | 51 | | var evaluationPolicy = EvaluationTimestampPolicyParser.ParseOrNull( |
| | 1 | 52 | | settings.EvaluationPolicyKind, |
| | 1 | 53 | | settings.EvaluationPolicyOffset); |
| | 1 | 54 | | var reconstructAtTimestamp = evaluationTime is not null || evaluationPolicy is not null; |
| | 1 | 55 | | var modelConfig = PredictionModelConfig.Create(settings.Model, settings.ReasoningEffort); |
| | | 56 | | |
| | 1 | 57 | | var storedMatch = await predictionRepository.GetStoredMatchAsync( |
| | 1 | 58 | | settings.HomeTeam, |
| | 1 | 59 | | settings.AwayTeam, |
| | 1 | 60 | | settings.Matchday!.Value, |
| | 1 | 61 | | reconstructAtTimestamp ? null : modelConfig, |
| | 1 | 62 | | reconstructAtTimestamp ? null : settings.CommunityContext); |
| | | 63 | | |
| | 1 | 64 | | if (storedMatch is null) |
| | | 65 | | { |
| | 1 | 66 | | _console.MarkupLine($"[red]Stored match not found on matchday {settings.Matchday}:[/] {Markup.Escape(set |
| | 1 | 67 | | return 1; |
| | | 68 | | } |
| | | 69 | | |
| | 1 | 70 | | var promptMatch = ExperimentArtifactSupport.RehydrateForPromptOutput(storedMatch); |
| | 1 | 71 | | var resolvedEvaluationTime = evaluationTime |
| | 1 | 72 | | ?? (evaluationPolicy is null ? null : EvaluationTimestampResolver.Resolve(promptMatch, evaluationPolicy) |
| | | 73 | | |
| | | 74 | | ReconstructedMatchPredictionPrompt? reconstructedPrompt; |
| | 1 | 75 | | if (resolvedEvaluationTime is null) |
| | | 76 | | { |
| | 1 | 77 | | reconstructedPrompt = await reconstructionService.ReconstructMatchPredictionPromptAsync( |
| | 1 | 78 | | promptMatch, |
| | 1 | 79 | | modelConfig, |
| | 1 | 80 | | settings.CommunityContext, |
| | 1 | 81 | | settings.WithJustification); |
| | | 82 | | } |
| | | 83 | | else |
| | | 84 | | { |
| | 0 | 85 | | var selection = MatchContextDocumentCatalog.ForMatch( |
| | 0 | 86 | | promptMatch.HomeTeam, |
| | 0 | 87 | | promptMatch.AwayTeam, |
| | 0 | 88 | | settings.CommunityContext); |
| | | 89 | | |
| | 0 | 90 | | reconstructedPrompt = await reconstructionService.ReconstructMatchPredictionPromptAtTimestampAsync( |
| | 0 | 91 | | promptMatch, |
| | 0 | 92 | | settings.Model, |
| | 0 | 93 | | settings.CommunityContext, |
| | 0 | 94 | | resolvedEvaluationTime.Value, |
| | 0 | 95 | | selection.RequiredDocumentNames, |
| | 0 | 96 | | selection.OptionalDocumentNames, |
| | 0 | 97 | | settings.WithJustification); |
| | | 98 | | } |
| | | 99 | | |
| | 1 | 100 | | if (reconstructedPrompt is null) |
| | | 101 | | { |
| | 0 | 102 | | _console.MarkupLine("[red]No stored prediction metadata found for that match/model/community combination |
| | 0 | 103 | | return 1; |
| | | 104 | | } |
| | | 105 | | |
| | 1 | 106 | | var outcomes = await matchOutcomeRepository.GetMatchdayOutcomesAsync( |
| | 1 | 107 | | settings.Matchday.Value, |
| | 1 | 108 | | settings.CommunityContext); |
| | | 109 | | |
| | 1 | 110 | | var outcome = outcomes.FirstOrDefault(candidate => |
| | 1 | 111 | | string.Equals(candidate.HomeTeam, settings.HomeTeam, StringComparison.OrdinalIgnoreCase) && |
| | 1 | 112 | | string.Equals(candidate.AwayTeam, settings.AwayTeam, StringComparison.OrdinalIgnoreCase)); |
| | | 113 | | |
| | 1 | 114 | | if (outcome is null) |
| | | 115 | | { |
| | 0 | 116 | | _console.MarkupLine("[red]No persisted match outcome was found for the selected match.[/]"); |
| | 0 | 117 | | return 1; |
| | | 118 | | } |
| | | 119 | | |
| | 1 | 120 | | if (!outcome.HasOutcome || outcome.HomeGoals is null || outcome.AwayGoals is null) |
| | | 121 | | { |
| | 1 | 122 | | _console.MarkupLine("[red]The selected match does not have a completed persisted outcome yet.[/]"); |
| | 1 | 123 | | return 1; |
| | | 124 | | } |
| | | 125 | | |
| | 1 | 126 | | var export = BuildExport(reconstructedPrompt, outcome); |
| | 1 | 127 | | var outputPath = ResolveOutputPath(settings, export.DatasetItem.Metadata); |
| | 1 | 128 | | Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); |
| | | 129 | | |
| | 1 | 130 | | await File.WriteAllTextAsync( |
| | 1 | 131 | | outputPath, |
| | 1 | 132 | | JsonSerializer.Serialize(export, OutputJsonOptions)); |
| | | 133 | | |
| | 1 | 134 | | _console.MarkupLine($"[green]Wrote experiment item:[/] [yellow]{Markup.Escape(outputPath)}[/]"); |
| | 1 | 135 | | _console.MarkupLine($"[blue]Dataset item id:[/] {Markup.Escape(export.DatasetItem.Id)}"); |
| | 1 | 136 | | _console.MarkupLine($"[blue]{(resolvedEvaluationTime is null ? "Prediction timestamp" : "Reconstruction time |
| | 1 | 137 | | _console.MarkupLine($"[blue]Outcome:[/] {outcome.HomeGoals}:{outcome.AwayGoals} ({outcome.Availability})"); |
| | | 138 | | |
| | 1 | 139 | | return 0; |
| | | 140 | | } |
| | 1 | 141 | | catch (Exception ex) |
| | | 142 | | { |
| | 1 | 143 | | _logger.LogError(ex, "Error exporting experiment item"); |
| | 1 | 144 | | _console.MarkupLine($"[red]Error:[/] {Markup.Escape(ex.Message)}"); |
| | 1 | 145 | | return 1; |
| | | 146 | | } |
| | 1 | 147 | | } |
| | | 148 | | |
| | | 149 | | private static ExportedExperimentItem BuildExport( |
| | | 150 | | ReconstructedMatchPredictionPrompt reconstructedPrompt, |
| | | 151 | | PersistedMatchOutcome outcome) |
| | | 152 | | { |
| | 1 | 153 | | using var matchJsonDocument = JsonDocument.Parse(reconstructedPrompt.MatchJson); |
| | 1 | 154 | | var tippSpielId = outcome.TippSpielId ?? throw new InvalidOperationException( |
| | 1 | 155 | | $"Persisted outcome for {outcome.HomeTeam} vs {outcome.AwayTeam} is missing tippspielId."); |
| | | 156 | | |
| | 1 | 157 | | var metadata = new MatchExperimentMetadata( |
| | 1 | 158 | | reconstructedPrompt.CommunityContext, |
| | 1 | 159 | | outcome.Competition, |
| | 1 | 160 | | reconstructedPrompt.Match.Matchday, |
| | 1 | 161 | | reconstructedPrompt.Match.HomeTeam, |
| | 1 | 162 | | reconstructedPrompt.Match.AwayTeam, |
| | 1 | 163 | | tippSpielId, |
| | 1 | 164 | | reconstructedPrompt.Model, |
| | 1 | 165 | | reconstructedPrompt.IncludeJustification, |
| | 1 | 166 | | reconstructedPrompt.PromptTimestamp, |
| | 1 | 167 | | reconstructedPrompt.PromptTemplatePath, |
| | 1 | 168 | | reconstructedPrompt.ContextDocumentNames, |
| | 1 | 169 | | reconstructedPrompt.ResolvedContextDocuments |
| | 1 | 170 | | .Select(document => new MatchExperimentResolvedContextDocument( |
| | 1 | 171 | | document.DocumentName, |
| | 1 | 172 | | document.Version, |
| | 1 | 173 | | document.CreatedAt)) |
| | 1 | 174 | | .ToList() |
| | 1 | 175 | | .AsReadOnly(), |
| | 1 | 176 | | new MatchExperimentOutcome( |
| | 1 | 177 | | outcome.HomeGoals!.Value, |
| | 1 | 178 | | outcome.AwayGoals!.Value)); |
| | | 179 | | |
| | 1 | 180 | | return new ExportedExperimentItem( |
| | 1 | 181 | | new MatchExperimentDatasetItem( |
| | 1 | 182 | | ExperimentArtifactSupport.BuildHostedDatasetItemId(outcome.Competition, outcome.CommunityContext, tippSp |
| | 1 | 183 | | matchJsonDocument.RootElement.Clone(), |
| | 1 | 184 | | new MatchExperimentExpectedOutput( |
| | 1 | 185 | | outcome.HomeGoals!.Value, |
| | 1 | 186 | | outcome.AwayGoals!.Value, |
| | 1 | 187 | | outcome.Availability.ToString()), |
| | 1 | 188 | | metadata), |
| | 1 | 189 | | new MatchExperimentRunnerPayload( |
| | 1 | 190 | | reconstructedPrompt.SystemPrompt, |
| | 1 | 191 | | reconstructedPrompt.MatchJson)); |
| | 1 | 192 | | } |
| | | 193 | | |
| | | 194 | | private static string ResolveOutputPath(ExportExperimentItemSettings settings, MatchExperimentMetadata metadata) |
| | | 195 | | { |
| | 1 | 196 | | if (!string.IsNullOrWhiteSpace(settings.OutputPath)) |
| | | 197 | | { |
| | 1 | 198 | | return Path.GetFullPath(settings.OutputPath); |
| | | 199 | | } |
| | | 200 | | |
| | 0 | 201 | | var fileName = $"{metadata.Matchday:00}-{ExperimentArtifactSupport.Slugify(metadata.HomeTeam)}-vs-{ExperimentArt |
| | 0 | 202 | | return Path.GetFullPath(Path.Combine("artifacts", "langfuse-experiments", "items", fileName)); |
| | | 203 | | } |
| | | 204 | | } |