| | | 1 | | using Intentum.Core.Behavior; |
| | | 2 | | using Intentum.Core.Contracts; |
| | | 3 | | using Intentum.Runtime.Engine; |
| | | 4 | | using Intentum.Runtime.Policy; |
| | | 5 | | |
| | | 6 | | namespace Intentum.Experiments; |
| | | 7 | | |
| | | 8 | | /// <summary> |
| | | 9 | | /// A/B experiment: run behavior spaces through multiple variants with deterministic |
| | | 10 | | /// hash-based traffic splitting and statistical significance reporting. |
| | | 11 | | /// </summary> |
| | | 12 | | public sealed class IntentExperiment |
| | | 13 | | { |
| | 5 | 14 | | private readonly List<ExperimentVariant> _variants = []; |
| | 5 | 15 | | private readonly List<int> _trafficSplit = []; |
| | | 16 | | |
| | | 17 | | /// <summary> |
| | | 18 | | /// Adds a variant (model + policy) to the experiment. |
| | | 19 | | /// </summary> |
| | | 20 | | public IntentExperiment AddVariant(string name, IIntentModel model, IntentPolicy policy) |
| | 6 | 21 | | { |
| | 6 | 22 | | _variants.Add(new ExperimentVariant(name, model ?? throw new ArgumentNullException(nameof(model)), |
| | 6 | 23 | | policy ?? throw new ArgumentNullException(nameof(policy)))); |
| | 5 | 24 | | return this; |
| | 5 | 25 | | } |
| | | 26 | | |
| | | 27 | | /// <summary> |
| | | 28 | | /// Sets traffic split percentages (e.g. 50, 50 for two variants). Must sum to 100 and match variant count. |
| | | 29 | | /// </summary> |
| | | 30 | | public IntentExperiment SplitTraffic(params int[]? percentages) |
| | 2 | 31 | | { |
| | 2 | 32 | | _trafficSplit.Clear(); |
| | 2 | 33 | | _trafficSplit.AddRange(percentages ?? []); |
| | 2 | 34 | | return this; |
| | 2 | 35 | | } |
| | | 36 | | |
| | | 37 | | /// <summary> |
| | | 38 | | /// Runs the experiment with deterministic hash-based variant assignment. |
| | | 39 | | /// Each behavior space is assigned a variant based on a stable hash of its identity, |
| | | 40 | | /// ensuring the same space always gets the same variant across runs. |
| | | 41 | | /// </summary> |
| | | 42 | | public Task<IReadOnlyList<ExperimentResult>> RunAsync( |
| | | 43 | | IReadOnlyList<BehaviorSpace> behaviorSpaces, |
| | | 44 | | CancellationToken cancellationToken = default) |
| | 5 | 45 | | { |
| | 5 | 46 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 5 | 47 | | if (_variants.Count == 0) |
| | 1 | 48 | | throw new InvalidOperationException("Add at least one variant with AddVariant."); |
| | | 49 | | |
| | 4 | 50 | | var split = NormalizeSplit(_trafficSplit, _variants.Count); |
| | 4 | 51 | | var results = new List<ExperimentResult>(); |
| | | 52 | | |
| | 60 | 53 | | foreach (var space in behaviorSpaces) |
| | 24 | 54 | | { |
| | 24 | 55 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 24 | 56 | | var identity = GetSpaceIdentity(space); |
| | 24 | 57 | | var variantIndex = SelectVariantByHash(identity, split); |
| | 24 | 58 | | var variant = _variants[variantIndex]; |
| | 24 | 59 | | var intent = variant.Model.Infer(space); |
| | 24 | 60 | | var decision = IntentPolicyEngine.Evaluate(intent, variant.Policy); |
| | 24 | 61 | | results.Add(new ExperimentResult(variant.Name, identity, intent, decision)); |
| | 24 | 62 | | } |
| | | 63 | | |
| | 4 | 64 | | return Task.FromResult<IReadOnlyList<ExperimentResult>>(results); |
| | 4 | 65 | | } |
| | | 66 | | |
| | | 67 | | /// <summary> |
| | | 68 | | /// Synchronous batch run. |
| | | 69 | | /// </summary> |
| | | 70 | | public IReadOnlyList<ExperimentResult> Run(IReadOnlyList<BehaviorSpace> behaviorSpaces) |
| | 2 | 71 | | => RunAsync(behaviorSpaces).GetAwaiter().GetResult(); |
| | | 72 | | |
| | | 73 | | /// <summary> |
| | | 74 | | /// Computes basic statistical significance between two variants using a chi-square test |
| | | 75 | | /// on the distribution of decisions. |
| | | 76 | | /// </summary> |
| | | 77 | | public static ExperimentSignificance ComputeSignificance( |
| | | 78 | | IReadOnlyList<ExperimentResult> results, |
| | | 79 | | string variantA, |
| | | 80 | | string variantB) |
| | 2 | 81 | | { |
| | 22 | 82 | | var groupA = results.Where(r => r.VariantName == variantA).ToList(); |
| | 22 | 83 | | var groupB = results.Where(r => r.VariantName == variantB).ToList(); |
| | | 84 | | |
| | 2 | 85 | | if (groupA.Count == 0 || groupB.Count == 0) |
| | 2 | 86 | | return new ExperimentSignificance(0, 1.0, false); |
| | | 87 | | |
| | 0 | 88 | | var allDecisions = groupA.Select(r => r.Decision) |
| | 0 | 89 | | .Concat(groupB.Select(r => r.Decision)) |
| | 0 | 90 | | .Distinct() |
| | 0 | 91 | | .ToList(); |
| | | 92 | | |
| | 0 | 93 | | var chiSquare = 0.0; |
| | 0 | 94 | | var totalA = (double)groupA.Count; |
| | 0 | 95 | | var totalB = (double)groupB.Count; |
| | 0 | 96 | | var total = totalA + totalB; |
| | | 97 | | |
| | 0 | 98 | | foreach (var decision in allDecisions) |
| | 0 | 99 | | { |
| | 0 | 100 | | var observedA = groupA.Count(r => r.Decision == decision); |
| | 0 | 101 | | var observedB = groupB.Count(r => r.Decision == decision); |
| | 0 | 102 | | var totalForDecision = observedA + observedB; |
| | | 103 | | |
| | 0 | 104 | | var expectedA = totalForDecision * (totalA / total); |
| | 0 | 105 | | var expectedB = totalForDecision * (totalB / total); |
| | | 106 | | |
| | 0 | 107 | | if (expectedA > 0) chiSquare += Math.Pow(observedA - expectedA, 2) / expectedA; |
| | 0 | 108 | | if (expectedB > 0) chiSquare += Math.Pow(observedB - expectedB, 2) / expectedB; |
| | 0 | 109 | | } |
| | | 110 | | |
| | 0 | 111 | | var degreesOfFreedom = Math.Max(1, allDecisions.Count - 1); |
| | 0 | 112 | | var pValue = ChiSquarePValue(chiSquare, degreesOfFreedom); |
| | | 113 | | |
| | 0 | 114 | | return new ExperimentSignificance(chiSquare, pValue, pValue < 0.05); |
| | 2 | 115 | | } |
| | | 116 | | |
| | | 117 | | private static string GetSpaceIdentity(BehaviorSpace space) |
| | 24 | 118 | | { |
| | 24 | 119 | | var entityId = space.GetMetadata<string>("entityId"); |
| | 24 | 120 | | if (!string.IsNullOrEmpty(entityId)) |
| | 0 | 121 | | return entityId; |
| | | 122 | | |
| | 24 | 123 | | var sessionId = space.GetMetadata<string>("sessionId"); |
| | 24 | 124 | | if (!string.IsNullOrEmpty(sessionId)) |
| | 0 | 125 | | return sessionId; |
| | | 126 | | |
| | 73 | 127 | | return string.Join("|", space.Events.Select(e => $"{e.Actor}:{e.Action}")); |
| | 24 | 128 | | } |
| | | 129 | | |
| | | 130 | | private static int SelectVariantByHash(string identity, int[] split) |
| | 24 | 131 | | { |
| | 24 | 132 | | var hash = StableHash(identity); |
| | 24 | 133 | | var bucket = (int)(((uint)hash) % 100); |
| | 24 | 134 | | var cumulative = 0; |
| | 90 | 135 | | for (var i = 0; i < split.Length; i++) |
| | 45 | 136 | | { |
| | 45 | 137 | | cumulative += split[i]; |
| | 45 | 138 | | if (bucket < cumulative) |
| | 24 | 139 | | return i; |
| | 21 | 140 | | } |
| | 0 | 141 | | return split.Length - 1; |
| | 24 | 142 | | } |
| | | 143 | | |
| | | 144 | | private static int StableHash(string input) |
| | 24 | 145 | | { |
| | | 146 | | unchecked |
| | 24 | 147 | | { |
| | 24 | 148 | | var hash = 5381; |
| | 1150 | 149 | | foreach (var c in input) |
| | 539 | 150 | | hash = ((hash << 5) + hash) + c; |
| | 24 | 151 | | return hash; |
| | | 152 | | } |
| | 24 | 153 | | } |
| | | 154 | | |
| | | 155 | | private static int[] NormalizeSplit(List<int> requested, int variantCount) |
| | 4 | 156 | | { |
| | 4 | 157 | | if (requested.Count == variantCount && requested.Sum() == 100) |
| | 2 | 158 | | return requested.ToArray(); |
| | 2 | 159 | | var equal = 100 / variantCount; |
| | 2 | 160 | | var remainder = 100 % variantCount; |
| | 2 | 161 | | var split = new int[variantCount]; |
| | 8 | 162 | | for (var i = 0; i < variantCount; i++) |
| | 2 | 163 | | split[i] = equal + (i < remainder ? 1 : 0); |
| | 2 | 164 | | return split; |
| | 4 | 165 | | } |
| | | 166 | | |
| | | 167 | | /// <summary> |
| | | 168 | | /// Approximation of chi-square p-value using the Wilson-Hilferty transformation. |
| | | 169 | | /// </summary> |
| | | 170 | | private static double ChiSquarePValue(double chiSquare, int degreesOfFreedom) |
| | 0 | 171 | | { |
| | 0 | 172 | | if (chiSquare <= 0 || degreesOfFreedom <= 0) return 1.0; |
| | | 173 | | |
| | 0 | 174 | | var k = degreesOfFreedom; |
| | 0 | 175 | | var z = Math.Pow(chiSquare / k, 1.0 / 3) - (1.0 - 2.0 / (9.0 * k)); |
| | 0 | 176 | | z /= Math.Sqrt(2.0 / (9.0 * k)); |
| | | 177 | | |
| | 0 | 178 | | var pValue = 0.5 * Erfc(z / Math.Sqrt(2)); |
| | 0 | 179 | | return Math.Clamp(pValue, 0, 1); |
| | 0 | 180 | | } |
| | | 181 | | |
| | | 182 | | private static double Erfc(double x) |
| | 0 | 183 | | { |
| | 0 | 184 | | var t = 1.0 / (1.0 + 0.5 * Math.Abs(x)); |
| | 0 | 185 | | var tau = t * Math.Exp( |
| | 0 | 186 | | -x * x - 1.26551223 + |
| | 0 | 187 | | t * (1.00002368 + |
| | 0 | 188 | | t * (0.37409196 + |
| | 0 | 189 | | t * (0.09678418 + |
| | 0 | 190 | | t * (-0.18628806 + |
| | 0 | 191 | | t * (0.27886807 + |
| | 0 | 192 | | t * (-1.13520398 + |
| | 0 | 193 | | t * (1.48851587 + |
| | 0 | 194 | | t * (-0.82215223 + |
| | 0 | 195 | | t * 0.17087277))))))))); |
| | 0 | 196 | | return x >= 0 ? tau : 2 - tau; |
| | 0 | 197 | | } |
| | | 198 | | } |
| | | 199 | | |
| | | 200 | | /// <summary> |
| | | 201 | | /// Result of statistical significance test between two experiment variants. |
| | | 202 | | /// </summary> |
| | | 203 | | public sealed record ExperimentSignificance( |
| | | 204 | | double ChiSquare, |
| | | 205 | | double PValue, |
| | | 206 | | bool IsSignificant |
| | | 207 | | ); |