Custom LLM Provider for Foundation Models

import FoundationModels

// MARK: - Custom Model Configuration (the cache key)
struct EchoModelConfiguration: LanguageModelExecutorConfiguration, Hashable {
    let maxTokens: Int
}

// MARK: - The Executor: where inference actually runs
final class EchoModelExecutor: LanguageModelExecutor {
    typealias Configuration = EchoModelConfiguration

    private let configuration: EchoModelConfiguration

    required init(configuration: EchoModelConfiguration) {
        self.configuration = configuration
    }

    func prewarm() async {
        // Load weights, open connections, etc.
        // No-op for this echo model.
    }

    func respond(
        to request: LanguageModelRequest,
        model: EchoLanguageModel,
        channel: LanguageModelResponseChannel
    ) async throws {
        // 1. Send metadata first so callers can log/track the request.
        await channel.send(.metadataUpdate(.init(
            modelID: "echo-model-v1",
            requestID: UUID().uuidString
        )))

        // 2. Count prompt tokens from the transcript.
        let promptTokenCount = request.transcript.entries.reduce(0) { count, entry in
            if case .prompt(let content) = entry { return count + content.text.split(separator: " ").count }
            return count
        }
        await channel.send(.usageUpdate(.init(promptTokenCount: promptTokenCount)))

        // 3. Stream the response token-by-token.
        let lastPrompt: String = request.transcript.entries.compactMap { entry -> String? in
            if case .prompt(let content) = entry { return content.text }
            return nil
        }.last ?? "(nothing)"

        let reply = "Echo: \(lastPrompt)"
        for word in reply.split(separator: " ") {
            await channel.send(.textDelta(String(word) + " "))
        }
    }
}

// MARK: - The LanguageModel: describes capabilities & provides configuration
struct EchoLanguageModel: LanguageModel {
    typealias Executor = EchoModelExecutor

    var capabilities: LanguageModelCapabilities {
        LanguageModelCapabilities(supportsToolCalling: false, supportsVision: false)
    }

    func makeConfiguration() -> EchoModelConfiguration {
        EchoModelConfiguration(maxTokens: 256)
    }
}

// MARK: - Usage: identical to the built-in system model
@main
struct EchoProviderDemo {
    static func main() async throws {
        let model = EchoLanguageModel()
        let session = LanguageModelSession(model: model)
        let response = try await session.respond(to: "Hello, custom model!")
        print(response.content) // Echo: Hello, custom model!
    }
}