< Summary

Information
Class: Orchestrator.Commands.Observability.ExportExperimentAnalysis.PublishExperimentAnalysisCommand
Assembly: Orchestrator
File(s): /home/runner/work/KicktippAi/KicktippAi/src/Orchestrator/Commands/Observability/ExportExperimentAnalysis/PublishExperimentAnalysisCommand.cs
Line coverage
90%
Covered lines: 166
Uncovered lines: 18
Coverable lines: 184
Total lines: 272
Line coverage: 90.2%
Branch coverage
71%
Covered branches: 40
Total branches: 56
Branch coverage: 71.4%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Metrics

MethodBranch coverage Crap Score Cyclomatic complexity Line coverage
.ctor(...)100%11100%
ExecuteAsync()71.88%403280.43%
PostRunScoresAsync()100%11100%
BuildRunMetadata(...)100%88100%
DeriveExperimentName(...)66.67%66100%
TryResolveCompetition(...)50%44100%
TryResolveCommunityContext(...)50%44100%
ParseTimestampOrNull(...)50%22100%

File(s)

/home/runner/work/KicktippAi/KicktippAi/src/Orchestrator/Commands/Observability/ExportExperimentAnalysis/PublishExperimentAnalysisCommand.cs

#LineLine coverage
 1using System.Globalization;
 2using System.Text.Json;
 3using Microsoft.Extensions.Logging;
 4using Orchestrator.Commands.Observability.Experiments;
 5using Orchestrator.Infrastructure.Langfuse;
 6using Spectre.Console;
 7using Spectre.Console.Cli;
 8
 9namespace Orchestrator.Commands.Observability.ExportExperimentAnalysis;
 10
 11public sealed class PublishExperimentAnalysisCommand : AsyncCommand<PublishExperimentAnalysisSettings>
 12{
 13    private readonly IAnsiConsole _console;
 14    private readonly ILangfusePublicApiClient _langfuseClient;
 15    private readonly ILogger<PublishExperimentAnalysisCommand> _logger;
 16
 117    public PublishExperimentAnalysisCommand(
 118        IAnsiConsole console,
 119        ILangfusePublicApiClient langfuseClient,
 120        ILogger<PublishExperimentAnalysisCommand> logger)
 21    {
 122        _console = console;
 123        _langfuseClient = langfuseClient;
 124        _logger = logger;
 125    }
 26
 27    protected override async Task<int> ExecuteAsync(CommandContext context, PublishExperimentAnalysisSettings settings, 
 28    {
 29        try
 30        {
 131            var bundle = await PreparedExperimentCommandSupport.LoadJsonFileAsync<PreparedExperimentAnalysisBundle>(
 132                settings.InputPath,
 133                cancellationToken);
 134            var rowsByRunName = bundle.Rows
 135                .GroupBy(row => row.RunName, StringComparer.Ordinal)
 136                .ToDictionary(group => group.Key, group => group.ToList(), StringComparer.Ordinal);
 137            var experimentName = string.IsNullOrWhiteSpace(settings.ExperimentName)
 138                ? DeriveExperimentName(bundle)
 139                : settings.ExperimentName.Trim();
 140            var runResults = new List<object>();
 41
 142            PreparedExperimentSupport.ReportProgress(
 143                $"Publishing {bundle.Runs.Count} experiment run alias(es) from '{settings.InputPath}' to Langfuse Experi
 44
 145            foreach (var run in bundle.Runs.OrderBy(run => run.RunName, StringComparer.Ordinal))
 46            {
 147                if (!rowsByRunName.TryGetValue(run.RunName, out var rows) || rows.Count == 0)
 48                {
 049                    throw new InvalidOperationException($"Analysis bundle run '{run.RunName}' has no rows to publish.");
 50                }
 51
 152                var targetRunName = run.RunName + settings.RunNameSuffix;
 153                var runDescription = string.IsNullOrWhiteSpace(settings.Description)
 154                    ? $"Published from analysis bundle '{Path.GetFileName(settings.InputPath)}' for Langfuse Experiments
 155                    : settings.Description.Trim();
 56
 157                if (settings.DryRun)
 58                {
 059                    runResults.Add(new
 060                    {
 061                        sourceRunName = run.RunName,
 062                        targetRunName,
 063                        rowCount = rows.Count,
 064                        experimentName
 065                    });
 066                    continue;
 67                }
 68
 169                var existingRun = await _langfuseClient.GetDatasetRunAsync(
 170                    bundle.DatasetName,
 171                    targetRunName,
 172                    cancellationToken);
 73
 174                if (existingRun is not null)
 75                {
 076                    if (!settings.ReplaceRuns)
 77                    {
 078                        throw new InvalidOperationException(
 079                            $"Published run alias '{targetRunName}' already exists in dataset '{bundle.DatasetName}'. Us
 80                    }
 81
 082                    await _langfuseClient.DeleteDatasetRunAsync(bundle.DatasetName, targetRunName, cancellationToken);
 83                }
 84
 185                var publishedAtUtc = ExperimentArtifactSupport.FormatStartedAtUtc(DateTimeOffset.UtcNow);
 186                var runMetadata = BuildRunMetadata(bundle, run, rows);
 187                var metadata = PreparedExperimentSupport.BuildLangfuseExperimentMetadata(
 188                    runMetadata,
 189                    experimentName,
 190                    targetRunName,
 191                    new Dictionary<string, string?>
 192                    {
 193                        ["sourceRunName"] = run.RunName,
 194                        ["sourceDatasetRunId"] = run.DatasetRunId,
 195                        ["publishedFromAnalysisBundle"] = Path.GetFileName(settings.InputPath),
 196                        ["publishedAtUtc"] = publishedAtUtc
 197                    });
 198                var createdAt = ParseTimestampOrNull(run.StartedAtUtc);
 199                string? targetDatasetRunId = null;
 100
 1101                PreparedExperimentSupport.ReportProgress(
 1102                    $"Publishing alias '{targetRunName}' with {rows.Count} item(s).");
 103
 1104                foreach (var row in rows.OrderBy(row => row.DatasetItemId, StringComparer.Ordinal))
 105                {
 1106                    var datasetRunItem = await _langfuseClient.CreateDatasetRunItemAsync(
 1107                        new LangfuseCreateDatasetRunItemRequest(
 1108                            targetRunName,
 1109                            row.DatasetItemId,
 1110                            row.TraceId,
 1111                            runDescription,
 1112                            metadata,
 1113                            row.ObservationId,
 1114                            createdAt),
 1115                        cancellationToken);
 116
 1117                    targetDatasetRunId ??= datasetRunItem.DatasetRunId;
 118                }
 119
 1120                if (string.IsNullOrWhiteSpace(targetDatasetRunId))
 121                {
 0122                    throw new InvalidOperationException($"Publishing run alias '{targetRunName}' did not return a datase
 123                }
 124
 1125                await PostRunScoresAsync(targetDatasetRunId, run, metadata, cancellationToken);
 126
 1127                runResults.Add(new
 1128                {
 1129                    sourceRunName = run.RunName,
 1130                    targetRunName,
 1131                    datasetRunId = targetDatasetRunId,
 1132                    rowCount = rows.Count,
 1133                    experimentName
 1134                });
 1135            }
 136
 1137            var summary = new
 1138            {
 1139                bundle.DatasetName,
 1140                bundle.TaskType,
 1141                experimentName,
 1142                dryRun = settings.DryRun,
 1143                runCount = runResults.Count,
 1144                runs = runResults
 1145            };
 146
 1147            _console.WriteLine(JsonSerializer.Serialize(summary, PreparedExperimentCommandSupport.JsonOptions));
 1148            return 0;
 149        }
 0150        catch (Exception ex)
 151        {
 0152            _logger.LogError(ex, "Error publishing experiment analysis bundle");
 0153            _console.MarkupLine($"[red]Error:[/] {Markup.Escape(ex.Message)}");
 0154            return 1;
 155        }
 1156    }
 157
 158    private async Task PostRunScoresAsync(
 159        string datasetRunId,
 160        PreparedExperimentAnalysisRun run,
 161        JsonElement metadata,
 162        CancellationToken cancellationToken)
 163    {
 1164        await _langfuseClient.CreateScoreAsync(
 1165            new LangfuseCreateScoreRequest(
 1166                "total_kicktipp_points",
 1167                run.AggregateScores.TotalKicktippPoints,
 1168                DatasetRunId: datasetRunId,
 1169                Comment: $"Published aggregate score for {run.RowCount} item(s)",
 1170                Id: PreparedExperimentSupport.CreateScoreId("total_kicktipp_points", datasetRunId),
 1171                Metadata: metadata),
 1172            cancellationToken);
 173
 1174        await _langfuseClient.CreateScoreAsync(
 1175            new LangfuseCreateScoreRequest(
 1176                "avg_kicktipp_points",
 1177                run.AggregateScores.AvgKicktippPoints,
 1178                DatasetRunId: datasetRunId,
 1179                Comment: $"Published aggregate score for {run.RowCount} item(s)",
 1180                Id: PreparedExperimentSupport.CreateScoreId("avg_kicktipp_points", datasetRunId),
 1181                Metadata: metadata),
 1182            cancellationToken);
 1183    }
 184
 185    private static PreparedExperimentRunMetadata BuildRunMetadata(
 186        PreparedExperimentAnalysisBundle bundle,
 187        PreparedExperimentAnalysisRun run,
 188        IReadOnlyList<PreparedExperimentAnalysisRow> rows)
 189    {
 1190        var datasetItemIdMap = rows
 1191            .GroupBy(row => row.SourceDatasetItemId, StringComparer.Ordinal)
 1192            .Where(group => group.Select(row => row.DatasetItemId).Distinct(StringComparer.Ordinal).Count() == 1)
 1193            .ToDictionary(group => group.Key, group => group.First().DatasetItemId, StringComparer.Ordinal);
 194
 1195        return new PreparedExperimentRunMetadata
 1196        {
 1197            Runner = "experiment-analysis-publisher",
 1198            TaskType = run.TaskType,
 1199            CommunityContext = TryResolveCommunityContext(bundle.DatasetName),
 1200            Competition = TryResolveCompetition(bundle.DatasetName),
 1201            DatasetName = bundle.DatasetName,
 1202            PromptKey = run.PromptKey,
 1203            ReasoningEffort = run.ReasoningEffort,
 1204            SliceKind = run.SliceKind,
 1205            SliceKey = run.SliceKey,
 1206            SourcePoolKey = run.SourcePoolKey,
 1207            SelectedItemIdsHash = run.SelectedItemIdsHash,
 1208            SelectedItemIdsCount = run.SelectedItemIdsCount,
 1209            SampleSize = run.SampleSize,
 1210            EvaluationTimestampPolicyKey = run.EvaluationTimestampPolicyKey,
 1211            EvaluationTime = run.EvaluationTime,
 1212            StartedAtUtc = run.StartedAtUtc,
 1213            IncludeJustification = false,
 1214            PromptVersion = run.PromptKey,
 1215            SourceDatasetKind = run.TaskType,
 1216            DatasetItemIdMap = datasetItemIdMap,
 1217            Model = run.Model,
 1218            RunSubjectKind = run.RunSubjectKind,
 1219            RunSubjectId = run.RunSubjectId,
 1220            RunSubjectDisplayName = run.RunSubjectDisplayName,
 1221            BatchStrategy = "published-analysis"
 1222        };
 223    }
 224
 225    private static string DeriveExperimentName(PreparedExperimentAnalysisBundle bundle)
 226    {
 1227        var datasetSegments = bundle.DatasetName
 1228            .Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
 1229        var community = datasetSegments.Length >= 3
 1230            ? datasetSegments[2]
 1231            : "unknown-community";
 1232        var datasetTail = datasetSegments.Length > 0
 1233            ? datasetSegments[^1]
 1234            : "analysis";
 235
 1236        return string.Join(
 1237            "__",
 1238            new[]
 1239            {
 1240                bundle.TaskType,
 1241                community,
 1242                datasetTail
 1243            }.Select(ExperimentArtifactSupport.Slugify));
 244    }
 245
 246    private static string? TryResolveCompetition(string datasetName)
 247    {
 1248        var datasetSegments = datasetName.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntr
 1249        return datasetSegments.Length >= 2 && string.Equals(datasetSegments[0], "match-predictions", StringComparison.Or
 1250            ? datasetSegments[1]
 1251            : null;
 252    }
 253
 254    private static string? TryResolveCommunityContext(string datasetName)
 255    {
 1256        var datasetSegments = datasetName.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntr
 1257        return datasetSegments.Length >= 3 && string.Equals(datasetSegments[0], "match-predictions", StringComparison.Or
 1258            ? datasetSegments[2]
 1259            : null;
 260    }
 261
 262    private static DateTimeOffset? ParseTimestampOrNull(string? value)
 263    {
 1264        return DateTimeOffset.TryParse(
 1265            value,
 1266            CultureInfo.InvariantCulture,
 1267            DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
 1268            out var parsed)
 1269            ? parsed
 1270            : null;
 271    }
 272}
Generated by: ReportGenerator 5.5.10.0
06/15/2026 - 23:01:29
GitHub | reportgenerator.io

Methods/Properties

.ctor(Spectre.Console.IAnsiConsole, Orchestrator.Infrastructure.Langfuse.ILangfusePublicApiClient, Microsoft.Extensions.Logging.ILogger<Orchestrator.Commands.Observability.ExportExperimentAnalysis.PublishExperimentAnalysisCommand>)
ExecuteAsync()
PostRunScoresAsync()
BuildRunMetadata(Orchestrator.Commands.Observability.Experiments.PreparedExperimentAnalysisBundle, Orchestrator.Commands.Observability.Experiments.PreparedExperimentAnalysisRun, System.Collections.Generic.IReadOnlyList<Orchestrator.Commands.Observability.Experiments.PreparedExperimentAnalysisRow>)
DeriveExperimentName(Orchestrator.Commands.Observability.Experiments.PreparedExperimentAnalysisBundle)
TryResolveCompetition(string)
TryResolveCommunityContext(string)
ParseTimestampOrNull(string)