Evaluations Framework – Synthetic Data Generation & Agentic Evaluation

import Evaluations
import FoundationModels

// MARK: – Domain model

@Generable
struct BookSample: Sendable {
    @Guide(description: "A written review of the book, at least 100 characters long.")
    var review: String
    @Guide(description: "3–8 lowercase genre/mood tags, e.g. [\"gothic\", \"suspense\"]")
    var tags: [String]
}

// MARK: – Seed dataset

let seedSamples: [ModelSample<String, [String]>] = [
    ModelSample(
        prompt: "A sweeping love story set in 19th-century England.",
        expectedOutput: ["romance", "classic", "social-commentary"]
    ),
    ModelSample(
        prompt: "A scientist creates life and faces the consequences.",
        expectedOutput: ["gothic", "horror", "philosophy"]
    ),
]

// MARK: – Generator configuration

let generator = SampleGenerator<String, [String]>(
    prompt: """
    Generate diverse book review samples covering a wide range of genres,
    moods, and tones. Reviews must be at least 100 characters long and
    vary in length. Each sample must include between 3 and 8 lowercase tags.
    """,
    dataset: seedSamples,
    targetCount: 100,
    sessionProvider: {
        // Swap in PrivateCloudComputeLanguageModel for larger context windows.
        let model = SystemLanguageModel.default
        let session = LanguageModelSession(model: model)
        return session
    },
    samplingStrategy: .random,
    validator: { sample in
        // Rule 1: review must be at least 100 characters
        guard sample.prompt.count >= 100 else { return .invalid(reason: "Review too short") }
        // Rule 2: between 3 and 8 tags
        guard (3...8).contains(sample.expectedOutput.count) else { return .invalid(reason: "Tag count out of range") }
        // Rule 3: all tags must be lowercase
        guard sample.expectedOutput.allSatisfy({ $0 == $0.lowercased() }) else { return .invalid(reason: "Tags must be lowercase") }
        return .valid
    }
)

// MARK: – Run generation

func buildExpandedDataset() async throws -> [ModelSample<String, [String]>] {
    var expandedDataset = seedSamples

    for try await sample in generator.run() {
        expandedDataset.append(sample)
    }

    let invalidCount = generator.invalidSamples.count
    print("Generation complete. Valid: \(expandedDataset.count), Invalid: \(invalidCount)")
    return expandedDataset
}