Vision Tap-to-Segment API

import Vision
import UIKit
import CoreVideo

final class TapToSegmentViewController: UIViewController {

    // MARK: - Asset readiness

    func ensureModelReady() async throws {
        let status = GenerateIterativeSegmentationRequest.assetStatus
        switch status {
        case .ready:
            print("Model already downloaded.")
        case .notDownloaded, .downloading:
            print("Downloading segmentation model…")
            try await GenerateIterativeSegmentationRequest.downloadAssets()
        default:
            break
        }
    }

    // MARK: - Segmentation

    /// `tapPoint` is in Vision normalized coordinates: origin bottom-left, values 0…1
    func segmentObject(
        in image: UIImage,
        at tapPoint: CGPoint
    ) async throws -> CVPixelBuffer? {

        guard let cgImage = image.cgImage else { return nil }

        let handler = ImageRequestHandler(cgImage)

        // Initial seed point — tap somewhere on the coffee cup
        var request = GenerateIterativeSegmentationRequest()
        request.seeds = [
            .point(tapPoint, phase: .include)
        ]

        let result = try await handler.perform(request)
        let initialMask: CVPixelBuffer = result.pixelBuffer
        print("Got initial mask: \(initialMask)")

        // Refine: also include a nearby point (e.g., the saucer plate)
        let refinementPoint = CGPoint(x: tapPoint.x, y: tapPoint.y - 0.08)
        request.seeds.append(
            .point(refinementPoint, phase: .include)
        )

        let refined = try await handler.perform(request)
        return refined.pixelBuffer
    }

    // MARK: - Exclusion example

    /// Remove a sub-region from the existing mask (e.g., exclude the cup, keep only coffee)
    func refineByExcluding(
        in image: UIImage,
        includePoint: CGPoint,
        excludePoint: CGPoint
    ) async throws -> CVPixelBuffer? {

        guard let cgImage = image.cgImage else { return nil }
        let handler = ImageRequestHandler(cgImage)

        var request = GenerateIterativeSegmentationRequest()
        request.seeds = [
            .point(includePoint, phase: .include),
            .point(excludePoint, phase: .exclude)
        ]

        let result = try await handler.perform(request)
        return result.pixelBuffer
    }
}