| | | 1 | | using EHonda.KicktippAi.Core; |
| | | 2 | | using Microsoft.Extensions.Logging; |
| | | 3 | | using NodaTime; |
| | | 4 | | using OpenAiIntegration; |
| | | 5 | | using Orchestrator.Commands.Observability; |
| | | 6 | | using Orchestrator.Infrastructure.Factories; |
| | | 7 | | using Spectre.Console; |
| | | 8 | | using Spectre.Console.Cli; |
| | | 9 | | |
| | | 10 | | namespace Orchestrator.Commands.Observability.ReconstructPrompt; |
| | | 11 | | |
| | | 12 | | /// <summary> |
| | | 13 | | /// Reconstructs historical prompt inputs for a stored match prediction. |
| | | 14 | | /// </summary> |
| | | 15 | | public class ReconstructPromptCommand : AsyncCommand<ReconstructPromptSettings> |
| | | 16 | | { |
| | 1 | 17 | | private static readonly DateTimeZone BundesligaTimeZone = DateTimeZoneProviders.Tzdb["Europe/Berlin"]; |
| | | 18 | | |
| | | 19 | | private readonly IAnsiConsole _console; |
| | | 20 | | private readonly IFirebaseServiceFactory _firebaseServiceFactory; |
| | | 21 | | private readonly ILogger<ReconstructPromptCommand> _logger; |
| | | 22 | | |
| | 1 | 23 | | public ReconstructPromptCommand( |
| | 1 | 24 | | IAnsiConsole console, |
| | 1 | 25 | | IFirebaseServiceFactory firebaseServiceFactory, |
| | 1 | 26 | | ILogger<ReconstructPromptCommand> logger) |
| | | 27 | | { |
| | 1 | 28 | | _console = console; |
| | 1 | 29 | | _firebaseServiceFactory = firebaseServiceFactory; |
| | 1 | 30 | | _logger = logger; |
| | 1 | 31 | | } |
| | | 32 | | |
| | | 33 | | public override async Task<int> ExecuteAsync(CommandContext context, ReconstructPromptSettings settings) |
| | | 34 | | { |
| | | 35 | | try |
| | | 36 | | { |
| | 1 | 37 | | _console.MarkupLine($"[green]Reconstructing prompt for:[/] [yellow]{Markup.Escape(settings.HomeTeam)}[/] vs |
| | | 38 | | |
| | 1 | 39 | | var predictionRepository = _firebaseServiceFactory.CreatePredictionRepository(); |
| | 1 | 40 | | var contextRepository = _firebaseServiceFactory.CreateContextRepository(); |
| | 1 | 41 | | var reconstructionService = new MatchPromptReconstructionService( |
| | 1 | 42 | | predictionRepository, |
| | 1 | 43 | | contextRepository, |
| | 1 | 44 | | new InstructionsTemplateProvider(PromptsFileProvider.Create())); |
| | | 45 | | |
| | 1 | 46 | | var evaluationTime = EvaluationTimeParser.ParseOrNull(settings.EvaluationTime); |
| | | 47 | | |
| | 1 | 48 | | var match = await ResolveMatchAsync(predictionRepository, settings, evaluationTime is not null); |
| | 1 | 49 | | if (match is null) |
| | | 50 | | { |
| | 1 | 51 | | _console.MarkupLine($"[red]Match not found on matchday {settings.Matchday}:[/] {Markup.Escape(settings.H |
| | 1 | 52 | | return 1; |
| | | 53 | | } |
| | | 54 | | |
| | 1 | 55 | | match = RehydrateForPromptOutput(match); |
| | | 56 | | |
| | | 57 | | ReconstructedMatchPredictionPrompt? reconstructedPrompt; |
| | 1 | 58 | | if (evaluationTime is null) |
| | | 59 | | { |
| | 1 | 60 | | reconstructedPrompt = await reconstructionService.ReconstructMatchPredictionPromptAsync( |
| | 1 | 61 | | match, |
| | 1 | 62 | | settings.Model, |
| | 1 | 63 | | settings.CommunityContext, |
| | 1 | 64 | | settings.WithJustification); |
| | | 65 | | } |
| | | 66 | | else |
| | | 67 | | { |
| | 1 | 68 | | var selection = MatchContextDocumentCatalog.ForMatch( |
| | 1 | 69 | | match.HomeTeam, |
| | 1 | 70 | | match.AwayTeam, |
| | 1 | 71 | | settings.CommunityContext); |
| | | 72 | | |
| | 1 | 73 | | reconstructedPrompt = await reconstructionService.ReconstructMatchPredictionPromptAtTimestampAsync( |
| | 1 | 74 | | match, |
| | 1 | 75 | | settings.Model, |
| | 1 | 76 | | settings.CommunityContext, |
| | 1 | 77 | | evaluationTime.Value, |
| | 1 | 78 | | selection.RequiredDocumentNames, |
| | 1 | 79 | | selection.OptionalDocumentNames, |
| | 1 | 80 | | settings.WithJustification); |
| | | 81 | | } |
| | | 82 | | |
| | 1 | 83 | | if (reconstructedPrompt is null) |
| | | 84 | | { |
| | 0 | 85 | | _console.MarkupLine("[red]No stored prediction metadata found for that match/model/community combination |
| | 0 | 86 | | return 1; |
| | | 87 | | } |
| | | 88 | | |
| | 1 | 89 | | var timestampLabel = evaluationTime is null ? "Prediction timestamp" : "Reconstruction timestamp"; |
| | 1 | 90 | | _console.MarkupLine($"[blue]{timestampLabel}:[/] {reconstructedPrompt.PromptTimestamp:O}"); |
| | 1 | 91 | | _console.MarkupLine($"[blue]Prompt template:[/] {Markup.Escape(reconstructedPrompt.PromptTemplatePath)}"); |
| | 1 | 92 | | _console.MarkupLine($"[blue]Justification variant:[/] {reconstructedPrompt.IncludeJustification}"); |
| | 1 | 93 | | _console.WriteLine(); |
| | 1 | 94 | | _console.MarkupLine("[green]Resolved context versions:[/]"); |
| | | 95 | | |
| | 1 | 96 | | foreach (var document in reconstructedPrompt.ResolvedContextDocuments) |
| | | 97 | | { |
| | 1 | 98 | | _console.MarkupLine($"[dim]- {Markup.Escape(document.DocumentName)} | v{document.Version} | {document.Cr |
| | | 99 | | } |
| | | 100 | | |
| | 1 | 101 | | _console.WriteLine(); |
| | 1 | 102 | | _console.MarkupLine("[green]Match JSON:[/]"); |
| | 1 | 103 | | _console.WriteLine(reconstructedPrompt.MatchJson); |
| | 1 | 104 | | _console.WriteLine(); |
| | 1 | 105 | | _console.MarkupLine("[green]System prompt:[/]"); |
| | 1 | 106 | | _console.WriteLine(reconstructedPrompt.SystemPrompt); |
| | | 107 | | |
| | 1 | 108 | | return 0; |
| | | 109 | | } |
| | 0 | 110 | | catch (Exception ex) |
| | | 111 | | { |
| | 0 | 112 | | _logger.LogError(ex, "Error executing reconstruct-prompt command"); |
| | 0 | 113 | | _console.MarkupLine($"[red]Error:[/] {Markup.Escape(ex.Message)}"); |
| | 0 | 114 | | return 1; |
| | | 115 | | } |
| | 1 | 116 | | } |
| | | 117 | | |
| | | 118 | | private static async Task<Match?> ResolveMatchAsync( |
| | | 119 | | IPredictionRepository predictionRepository, |
| | | 120 | | ReconstructPromptSettings settings, |
| | | 121 | | bool allowExactTimestampFallback) |
| | | 122 | | { |
| | 1 | 123 | | return await predictionRepository.GetStoredMatchAsync( |
| | 1 | 124 | | settings.HomeTeam, |
| | 1 | 125 | | settings.AwayTeam, |
| | 1 | 126 | | settings.Matchday!.Value, |
| | 1 | 127 | | allowExactTimestampFallback ? null : settings.Model, |
| | 1 | 128 | | allowExactTimestampFallback ? null : settings.CommunityContext); |
| | 1 | 129 | | } |
| | | 130 | | |
| | | 131 | | private static Match RehydrateForPromptOutput(Match match) |
| | | 132 | | { |
| | 1 | 133 | | var instant = match.StartsAt.ToInstant(); |
| | 1 | 134 | | var offset = BundesligaTimeZone.GetUtcOffset(instant); |
| | 1 | 135 | | var localizedStartsAt = instant.InZone(DateTimeZone.ForOffset(offset)); |
| | 1 | 136 | | return new Match(match.HomeTeam, match.AwayTeam, localizedStartsAt, match.Matchday, match.IsCancelled); |
| | | 137 | | } |
| | | 138 | | } |