< Summary

Information
Class: Orchestrator.Commands.Observability.Experiments.PreparedExperimentCommandSupport<T>
Assembly: Orchestrator
File(s): /home/runner/work/KicktippAi/KicktippAi/src/Orchestrator/Commands/Observability/Experiments/PreparedExperimentCommandSupport.cs
Line coverage
88%
Covered lines: 111
Uncovered lines: 15
Coverable lines: 126
Total lines: 215
Line coverage: 88%
Branch coverage
46%
Covered branches: 52
Total branches: 112
Branch coverage: 46.4%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Metrics

MethodBranch coverage Crap Score Cyclomatic complexity Line coverage
.cctor()100%11100%
LoadJsonFileAsync()50%22100%
NormalizeRunMetadata(...)42.86%848497.26%
ParseExplicitEvaluationTime(...)75%4488.89%
ParseEvaluationTimestampPolicy(...)50%4472.73%
ValidateManifest(...)56.25%321660%
EnsureTaskType(...)50%2260%

File(s)

/home/runner/work/KicktippAi/KicktippAi/src/Orchestrator/Commands/Observability/Experiments/PreparedExperimentCommandSupport.cs

#LineLine coverage
 1using System.Globalization;
 2using System.Text.Json;
 3using System.Text.Json.Serialization;
 4
 5namespace Orchestrator.Commands.Observability.Experiments;
 6
 7internal sealed record PreparedExperimentRunOptions(
 8    string Model,
 9    string PromptKey,
 10    bool IncludeJustification,
 11    string? EvaluationTime,
 12    string? EvaluationPolicyKind,
 13    string? EvaluationPolicyOffset,
 14    string? DatasetName,
 15    string PromptSource,
 16    string? LangfusePromptName,
 17    string? LangfusePromptLabel,
 18    int? LangfusePromptVersion,
 19    string BatchStrategy,
 20    int? BatchSize = null,
 21    int? BatchCount = null,
 22    string? ReasoningEffort = null,
 23    int? MaxOutputTokenCount = null,
 24    int? Parallelism = null);
 25
 26internal static class PreparedExperimentCommandSupport
 27{
 128    internal static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
 129    {
 130        PropertyNameCaseInsensitive = true,
 131        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
 132        WriteIndented = true,
 133        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
 134    };
 35
 36    public static async Task<T> LoadJsonFileAsync<T>(string path, CancellationToken cancellationToken)
 37    {
 138        var absolutePath = Path.GetFullPath(path);
 139        var raw = await File.ReadAllTextAsync(absolutePath, cancellationToken);
 140        var value = JsonSerializer.Deserialize<T>(raw, JsonOptions);
 141        return value ?? throw new InvalidOperationException($"JSON file '{absolutePath}' could not be deserialized.");
 142    }
 43
 44    public static PreparedExperimentRunMetadata NormalizeRunMetadata(
 45        PreparedExperimentRunMetadata runMetadata,
 46        PreparedExperimentManifest manifest,
 47        PreparedExperimentRunOptions options)
 48    {
 149        if (!string.IsNullOrWhiteSpace(runMetadata.Model)
 150            && !string.Equals(runMetadata.Model, options.Model, StringComparison.Ordinal))
 51        {
 052            throw new InvalidOperationException(
 053                $"Run metadata model '{runMetadata.Model}' does not match requested model '{options.Model}'.");
 54        }
 55
 156        var normalizedReasoningEffort = string.IsNullOrWhiteSpace(runMetadata.ReasoningEffort)
 157            ? options.ReasoningEffort
 158            : runMetadata.ReasoningEffort.Trim().ToLowerInvariant();
 159        var runSubjectId = string.IsNullOrWhiteSpace(runMetadata.RunSubjectId)
 160            ? string.IsNullOrWhiteSpace(normalizedReasoningEffort)
 161                ? options.Model
 162                : $"{options.Model}:reasoning-effort:{normalizedReasoningEffort}"
 163            : runMetadata.RunSubjectId;
 164        var runSubjectDisplayName = string.IsNullOrWhiteSpace(runMetadata.RunSubjectDisplayName)
 165            ? PreparedExperimentSupport.BuildRunSubjectDisplayName(options.Model, normalizedReasoningEffort)
 166            : runMetadata.RunSubjectDisplayName;
 67
 168        return runMetadata with
 169        {
 170            Runner = string.IsNullOrWhiteSpace(runMetadata.Runner) ? "match-experiment-runner" : runMetadata.Runner,
 171            TaskType = string.IsNullOrWhiteSpace(runMetadata.TaskType)
 172                ? PreparedExperimentSupport.ResolveTaskType(manifest)
 173                : runMetadata.TaskType,
 174            CommunityContext = string.IsNullOrWhiteSpace(runMetadata.CommunityContext)
 175                ? manifest.CommunityContext
 176                : runMetadata.CommunityContext,
 177            Model = options.Model,
 178            Competition = string.IsNullOrWhiteSpace(runMetadata.Competition) ? manifest.Competition : runMetadata.Compet
 179            SourceDatasetName = string.IsNullOrWhiteSpace(runMetadata.SourceDatasetName)
 180                ? manifest.SourceDatasetName
 181                : runMetadata.SourceDatasetName,
 182            DatasetName = string.IsNullOrWhiteSpace(runMetadata.DatasetName)
 183                ? options.DatasetName ?? manifest.SliceDatasetName
 184                : runMetadata.DatasetName,
 185            PromptKey = string.IsNullOrWhiteSpace(runMetadata.PromptKey) ? options.PromptKey : runMetadata.PromptKey,
 186            PromptSource = string.IsNullOrWhiteSpace(runMetadata.PromptSource) ? options.PromptSource : runMetadata.Prom
 187            LangfusePromptName = string.IsNullOrWhiteSpace(runMetadata.LangfusePromptName) ? options.LangfusePromptName 
 188            LangfusePromptLabel = string.IsNullOrWhiteSpace(runMetadata.LangfusePromptLabel) ? options.LangfusePromptLab
 189            LangfusePromptVersion = runMetadata.LangfusePromptVersion ?? options.LangfusePromptVersion,
 190            ReasoningEffort = normalizedReasoningEffort,
 191            MaxOutputTokenCount = options.MaxOutputTokenCount ?? runMetadata.MaxOutputTokenCount,
 192            SliceKind = string.IsNullOrWhiteSpace(runMetadata.SliceKind) ? manifest.SliceKind : runMetadata.SliceKind,
 193            SliceKey = string.IsNullOrWhiteSpace(runMetadata.SliceKey) ? manifest.SliceKey : runMetadata.SliceKey,
 194            SourcePoolKey = string.IsNullOrWhiteSpace(runMetadata.SourcePoolKey) ? manifest.SourcePoolKey : runMetadata.
 195            SelectedItemIdsCount = runMetadata.SelectedItemIdsCount > 0
 196                ? runMetadata.SelectedItemIdsCount
 197                : manifest.SelectedItemIds.Count > 0 ? manifest.SelectedItemIds.Count : manifest.Items.Count,
 198            SelectedItemIdsHash = string.IsNullOrWhiteSpace(runMetadata.SelectedItemIdsHash)
 199                ? string.IsNullOrWhiteSpace(manifest.SelectedItemIdsHash)
 1100                    ? ExperimentArtifactSupport.ComputeSelectedItemIdsHash(
 1101                        manifest.SelectedItemIds.Count > 0
 1102                            ? manifest.SelectedItemIds
 0103                            : manifest.Items.Select(item => item.SliceDatasetItemId))
 1104                    : manifest.SelectedItemIdsHash
 1105                : runMetadata.SelectedItemIdsHash,
 1106            SampleSize = runMetadata.SampleSize > 0 ? runMetadata.SampleSize : manifest.SampleSize > 0 ? manifest.Sample
 1107            MatchCount = runMetadata.MatchCount ?? manifest.MatchCount,
 1108            Repetitions = runMetadata.Repetitions ?? manifest.Repetitions,
 1109            SampleSeed = runMetadata.SampleSeed ?? manifest.SampleSeed,
 1110            SampleMethod = string.IsNullOrWhiteSpace(runMetadata.SampleMethod) ? manifest.SampleMethod : runMetadata.Sam
 1111            PromptVersion = string.IsNullOrWhiteSpace(runMetadata.PromptVersion)
 1112                ? string.IsNullOrWhiteSpace(runMetadata.PromptKey) ? options.PromptKey : runMetadata.PromptKey
 1113                : runMetadata.PromptVersion,
 1114            SourceDatasetKind = string.IsNullOrWhiteSpace(runMetadata.SourceDatasetKind)
 1115                ? PreparedExperimentSupport.ResolveTaskType(manifest)
 1116                : runMetadata.SourceDatasetKind,
 1117            DatasetItemIdMap = runMetadata.DatasetItemIdMap.Count > 0
 1118                ? runMetadata.DatasetItemIdMap
 1119                : PreparedExperimentSupport.CreateDatasetItemIdMap(manifest),
 1120            BatchStrategy = string.IsNullOrWhiteSpace(runMetadata.BatchStrategy) ? options.BatchStrategy : runMetadata.B
 1121            BatchSize = options.BatchSize ?? runMetadata.BatchSize,
 1122            BatchCount = options.BatchCount ?? runMetadata.BatchCount,
 1123            Parallelism = options.Parallelism ?? runMetadata.Parallelism,
 1124            RunSubjectId = runSubjectId,
 1125            RunSubjectDisplayName = runSubjectDisplayName
 1126        };
 127    }
 128
 129    public static DateTimeOffset? ParseExplicitEvaluationTime(PreparedExperimentRunMetadata runMetadata)
 130    {
 1131        if (string.IsNullOrWhiteSpace(runMetadata.EvaluationTime))
 132        {
 1133            return null;
 134        }
 135
 1136        if (DateTimeOffset.TryParse(
 1137                runMetadata.EvaluationTime,
 1138                CultureInfo.InvariantCulture,
 1139                DateTimeStyles.RoundtripKind,
 1140                out var parsedRoundtrip))
 141        {
 1142            return parsedRoundtrip;
 143        }
 144
 0145        return Commands.Observability.EvaluationTimeParser.Parse(runMetadata.EvaluationTime);
 146    }
 147
 148    public static EvaluationTimestampPolicy ParseEvaluationTimestampPolicy(PreparedExperimentRunMetadata runMetadata)
 149    {
 1150        if (runMetadata.EvaluationTimestampPolicy is null)
 151        {
 0152            throw new InvalidOperationException("Run metadata must contain evaluationTimestampPolicy.");
 153        }
 154
 1155        if (!string.Equals(
 1156                runMetadata.EvaluationTimestampPolicy.Reference,
 1157                EvaluationTimestampPolicy.StartsAtReference,
 1158                StringComparison.OrdinalIgnoreCase))
 159        {
 0160            throw new InvalidOperationException(
 0161                $"Evaluation policy reference must be '{EvaluationTimestampPolicy.StartsAtReference}'.");
 162        }
 163
 1164        return EvaluationTimestampPolicyParser.Parse(
 1165            runMetadata.EvaluationTimestampPolicy.Kind,
 1166            runMetadata.EvaluationTimestampPolicy.Offset);
 167    }
 168
 169    public static void ValidateManifest(PreparedExperimentManifest manifest)
 170    {
 1171        if (manifest.Items.Count == 0)
 172        {
 0173            throw new InvalidOperationException("Slice manifest must contain at least one item.");
 174        }
 175
 1176        var seenHostedIds = new HashSet<string>(StringComparer.Ordinal);
 1177        foreach (var item in manifest.Items)
 178        {
 1179            if (string.IsNullOrWhiteSpace(item.SourceDatasetItemId))
 180            {
 0181                throw new InvalidOperationException("Each slice manifest item must contain sourceDatasetItemId.");
 182            }
 183
 1184            if (string.IsNullOrWhiteSpace(item.SliceDatasetItemId))
 185            {
 0186                throw new InvalidOperationException("Each slice manifest item must contain sliceDatasetItemId.");
 187            }
 188
 1189            if (!seenHostedIds.Add(item.SliceDatasetItemId))
 190            {
 0191                throw new InvalidOperationException($"Duplicate slice dataset item id '{item.SliceDatasetItemId}' found 
 192            }
 193
 1194            if (string.IsNullOrWhiteSpace(item.HomeTeam) || string.IsNullOrWhiteSpace(item.AwayTeam))
 195            {
 0196                throw new InvalidOperationException("Each slice manifest item must contain non-empty homeTeam and awayTe
 197            }
 198
 1199            if (item.Matchday < 1)
 200            {
 0201                throw new InvalidOperationException($"Slice manifest item '{item.SliceDatasetItemId}' has an invalid mat
 202            }
 203        }
 1204    }
 205
 206    public static void EnsureTaskType(PreparedExperimentManifest manifest, string expectedTaskType)
 207    {
 1208        var actualTaskType = PreparedExperimentSupport.ResolveTaskType(manifest);
 1209        if (!string.Equals(actualTaskType, expectedTaskType, StringComparison.OrdinalIgnoreCase))
 210        {
 0211            throw new InvalidOperationException(
 0212                $"The manifest describes a '{actualTaskType}' dataset, but this command expects '{expectedTaskType}'.");
 213        }
 1214    }
 215}
Generated by: ReportGenerator 5.5.10.0
06/15/2026 - 23:01:29
GitHub | reportgenerator.io

Methods/Properties

.cctor()
LoadJsonFileAsync()
NormalizeRunMetadata(Orchestrator.Commands.Observability.Experiments.PreparedExperimentRunMetadata, Orchestrator.Commands.Observability.Experiments.PreparedExperimentManifest, Orchestrator.Commands.Observability.Experiments.PreparedExperimentRunOptions)
ParseExplicitEvaluationTime(Orchestrator.Commands.Observability.Experiments.PreparedExperimentRunMetadata)
ParseEvaluationTimestampPolicy(Orchestrator.Commands.Observability.Experiments.PreparedExperimentRunMetadata)
ValidateManifest(Orchestrator.Commands.Observability.Experiments.PreparedExperimentManifest)
EnsureTaskType(Orchestrator.Commands.Observability.Experiments.PreparedExperimentManifest, string)