El Bruno

ollama serve
ollama pull llama3.2

# From repo root
 dotnet run --project samples/VisionMemoryAgentSample -- --model-dir ./scripts/clip-models

IChatClient chatClient = new OllamaChatClient(new Uri("http://localhost:11434"), 
                                              modelId: ollamaModel)
    .AsBuilder()
    .UseFunctionInvocation()
    .Build();
var tools = new[]
{
    AIFunctionFactory.Create(IngestImage),
    AIFunctionFactory.Create(FindSimilarImages),
    AIFunctionFactory.Create(IngestImagesFromFolder)
};
AIAgent agent = chatClient.AsAIAgent(
    name: "VisionMemoryAgent",
    instructions: """
        You are a Vision Memory agent. You help users manage and search a local image collection.
        You have three tools:
        - IngestImage: to add an image to the in-memory store
        - IngestImagesFromFolder: to add all images from a folder to the in-memory store
        - FindSimilarImages: to search stored images using natural language
        Always use the tools when the user asks to ingest or search images.
        Report tool results clearly.
        """,
    tools: [.. tools]);
AgentSession session = await agent.CreateSessionAsync();

[Description("Ingest a local image file and store its CLIP embedding in memory. Returns confirmation.")]
string IngestImage(
    [Description("Absolute or relative path to the image file")] string path)
{
    if (!File.Exists(path))
        return $"Error: file not found: {path}";
    float[] embedding = imageEncoder.Encode(path);
    imageStore.Add((path, string.Empty, embedding));
    return $"Ingested '{Path.GetFileName(path)}'. Store now has {imageStore.Count} image(s).";
}

[Description("Ingest all the images from a local folder and store its CLIP embedding in memory. Returns confirmation.")]
string IngestImagesFromFolder(
    [Description("Absolute or relative path to the folder")] string folderPath)
{
    if (!Directory.Exists(folderPath))
        return $"Error: folder not found: {folderPath}";
    string[] extensions = [".jpg", ".jpeg", ".png", ".bmp", ".gif"];
    var imageFiles = Directory.GetFiles(folderPath)
        .Where(f => extensions.Contains(Path.GetExtension(f).ToLowerInvariant()))
        .ToList();
    if (imageFiles.Count == 0)
        return $"No images found in folder: {folderPath}";
    int ingestedCount = 0;
    foreach (string imagePath in imageFiles)
    {
        float[] embedding = imageEncoder.Encode(imagePath);
        imageStore.Add((imagePath, string.Empty, embedding));
        ingestedCount++;
    }
    return $"Ingested {ingestedCount} image(s) from '{folderPath}'. Store now has {imageStore.Count} image(s).";
}

[Description("Find images similar to a natural language query using CLIP embeddings. Returns top matches.")]
string FindSimilarImages(
    [Description("Natural language search query")] string query,
    [Description("Number of top results to return")] int topK = 3)
{
    if (imageStore.Count == 0)
        return "No images in store. Ingest some images first.";
    float[] queryEmbedding = textEncoder.Encode(query);
    var results = imageStore
        .Select(img => (img.Path, img.Tags, Score: TensorPrimitives.CosineSimilarity(
            queryEmbedding.AsSpan(), img.Embedding.AsSpan())))
        .OrderByDescending(r => r.Score)
        .Take(topK)
        .ToList();
    var lines = results.Select((r, i) =>
        $"  {i + 1}. {Path.GetFileName(r.Path)} (score: {r.Score:F4})");
    return $"Top {results.Count} result(s):\n{string.Join("\n", lines)}";
}

> Please ingest the image at ./samples/images/cat.jpg
Ingested 'cat.jpg'. Store now has 1 image(s).
> Ingest images from folder ./samples/images
Ingested 2 image(s) from './samples/images'. Store now has 3 image(s).
> Find images similar to "a beautiful sky at dusk"
Top 1 result(s):
  1. sunset.jpg (score: 0.2845)