| | | 1 | | namespace Intentum.AI.MultiModal; |
| | | 2 | | |
| | | 3 | | public static class MultiModalFusion |
| | | 4 | | { |
| | | 5 | | public static float[] Fuse(float[] behaviorEmbedding, IReadOnlyList<MultiModalInput> additionalInputs) |
| | 5 | 6 | | { |
| | 5 | 7 | | ArgumentNullException.ThrowIfNull(behaviorEmbedding); |
| | | 8 | | |
| | 4 | 9 | | if (additionalInputs.Count == 0) |
| | 2 | 10 | | return behaviorEmbedding; |
| | | 11 | | |
| | 2 | 12 | | var modalityEmbeddings = additionalInputs |
| | 3 | 13 | | .Where(i => i.Embedding != null) |
| | 3 | 14 | | .GroupBy(i => i.Modality) |
| | 2 | 15 | | .Select(g => |
| | 2 | 16 | | { |
| | 5 | 17 | | var embeddings = g.Select(i => i.Embedding!).ToList(); |
| | 2 | 18 | | var avg = new float[embeddings[0].Length]; |
| | 12 | 19 | | for (int j = 0; j < avg.Length; j++) |
| | 10 | 20 | | avg[j] = embeddings.Average(e => e[j]); |
| | 2 | 21 | | return avg; |
| | 2 | 22 | | }) |
| | 2 | 23 | | .ToList(); |
| | | 24 | | |
| | 4 | 25 | | var totalLength = behaviorEmbedding.Length + modalityEmbeddings.Sum(e => e.Length); |
| | 2 | 26 | | var result = new float[totalLength]; |
| | 2 | 27 | | behaviorEmbedding.CopyTo(result, 0); |
| | | 28 | | |
| | 2 | 29 | | var offset = behaviorEmbedding.Length; |
| | 10 | 30 | | foreach (var emb in modalityEmbeddings) |
| | 2 | 31 | | { |
| | 2 | 32 | | emb.CopyTo(result, offset); |
| | 2 | 33 | | offset += emb.Length; |
| | 2 | 34 | | } |
| | | 35 | | |
| | 2 | 36 | | return result; |
| | 4 | 37 | | } |
| | | 38 | | } |