Text Capture from Image - SwiftUI

Question

The new iOS 15 feature "Capture Text from Image" option comes only after second tap. So The Apple Team provided

let textFromCamera  = UIAction.captureTextFromCamera(responder: self.notes, identifier: nil)

When I tried to use same with SwiftUI, responser object should confirm to UIResponder/UIKeyInpunt.

How to use same with SwiftUI or Is there any alternate option in SwiftUI ?

Answer 1

Because I haven't found an answer for this anywhere, I had to figure it out myself. Doing that is actually possible in two ways.

SwiftUI

To create a button for this purpose, you need a class that adopts the protocols UIResponder and UIKeyInput. The required variable hasText and method deleteBackward are not used so you can just ignore them.

class ScanTextResponder: UIResponder, UIKeyInput {
    init(title: Binding<String>) {
        _title = title
    }

    @Binding var title: String

    var canCaptureTextFromCamera: Bool {
        canPerformAction(#selector(captureTextFromCamera(_:)), withSender: self)
    }

    var hasText = false

    func insertText(_ text: String) {
        title = text
    }

    func deleteBackward() {}
}

Because the feature does not work on older devices, you should add a condition like canCaptureTextFromCamera, to not show the button on these devices. Then you just need to use the responder class in a view like this:

struct ScanTextButton: View {
    init(title: Binding<String>) {
        self.responder = ScanTextResponder(title: title)
    }

    private let responder: ScanTextResponder

    var body: some View {
        if responder.canCaptureTextFromCamera {
            Button {
                responder.captureTextFromCamera(nil)

                // Allows dismissal of the camera after multiple presses
                let backup = responder.title
                responder.title = ""
                responder.title = backup
            } label: {
                Label("Scan Text", systemImage: "text.viewfinder")
            }
        }
    }
}

The last three lines in the button action are not necessary, but I have found that otherwise you can't dismiss the camera view after pressing the button multiple times while the camera is already visible. I haven't found a better way to do that.

UIViewRepresentable

Alternatively, there is also the possibility to integrate a UIViewRepresentable. The advantage of that is, that the button label and icon are preconfigured which includes localisation. The disadvantage is that you have to use UIKit if you want to style the button.

struct UIScanTextButton: UIViewRepresentable {
    let coordinator: ScanTextResponder

    func makeCoordinator() -> ScanTextResponder {
        coordinator
    }

    func makeUIView(context: Context) -> UIButton {
        UIButton(primaryAction: .captureTextFromCamera(responder: context.coordinator, identifier: nil))
    }

    func updateUIView(_ uiView: UIViewType, context: Context) {}
}

You can use that in your SwiftUI view like this:

UIScanTextButton(coordinator: responder)
            .fixedSize()

The fixedSize() modifier ensures that the view does not cover the whole screen.

Let me know if you have any questions.

Answer 2

I don't have an iPhone with iOS 15/neural engine to test it right now but this should work. I can get it to work up to clicking the menu.

The WWDC video states that it available in iPhone with neural engine which I think is the iPhone X and above. There is little documentation on this right now

import SwiftUI

@available(iOS 15.0, *)
struct ActionMenuView: View {
    @State var text: String = ""
    var body: some View {
        HStack{
            Text(text)
            ActionMenuView_UI(text: $text)
        }
    }
}
@available(iOS 15.0, *)
struct ActionMenuView_UI: UIViewRepresentable {
    @Binding var text: String
    
    func makeCoordinator() -> Coordinator {
        Coordinator(self)
    }
    func makeUIView(context: Context) -> some UIView {

        //let choosePhotoOrVideo = UIAction(…)
        //let takePhotoOrVideo = UIAction(…)
        //let scanDocuments = UIAction(…)

        // A UIButton can hold the menu, it is a long press to get it to come up
        let button = UIButton()
        let textFromCamera = UIAction.captureTextFromCamera(responder: context.coordinator, identifier: nil)
        let cameraMenu = UIMenu(children: [
            //choosePhotoOrVideo, takePhotoOrVideo, scanDocuments,
            textFromCamera])
        button.setImage(UIImage(systemName: "camera.badge.ellipsis"), for: .normal)
        button.menu = cameraMenu
        return button
    }
    func updateUIView(_ uiView: UIViewType, context: Context) {
        
    }
    //Making the Coordinator a UIResponder as! UIKeyInput gives access to the text
    class Coordinator: UIResponder, UIKeyInput{
        var hasText: Bool{
            !parent.text.isEmpty
        }
        
        let parent: ActionMenuView_UI
        
        init(_ parent: ActionMenuView_UI){
            self.parent = parent
        }
        func insertText(_ text: String) {
            //Update the @Binding
            parent.text = text
        }

        func deleteBackward() { }
    }
}
@available(iOS 15.0, *)
struct ActionMenuView_Previews: PreviewProvider {
    static var previews: some View {
        ActionMenuView()
    }
}