'is it possible to record video from the camera and simultaneously print the recorded frame number?

I am trying to write a swift 5 app that displays the current frame number when recording video from the camera. I'm quite new to Swift and have not been successful at finding a solution after searching for many days. I am looking for a way to invoke a function whenever a new frame arrives for recording, I do not require processing the image data, but for my application I would like to create a CSV file where each line corresponds to each recorded video frame exactly.

I thought func captureOutput gets called during the recording session but when I tested in my app it was not getting called.

In my view creates a new captureSession

let captureSession : AVCaptureSession = AVCaptureSession()

func beginSession() {
    
    if (captureSession.isRunning)
    {
        captureSession.stopRunning()
    }
    do
    {
        try captureInput = AVCaptureDeviceInput(device: captureDevice)
    
    ...
    captureSession.beginConfiguration()
    captureSession.addInput(captureInput)

and add the output to the session captureSession.addOutput(output).

    if captureSession.canAddOutput(videoOutput) {
        captureSession.addOutput(videoOutput)
        
        videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "video_frames_queue", qos: .userInteractive, attributes: [], autoreleaseFrequency: .workItem))
        videoOutput.alwaysDiscardsLateVideoFrames = true
    }

But after starting the app while recoding video I do not see captureOutput ever get called and "Record Frame Number: " never prints to the console.

func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
    print("Record Frame Number: ") //HOW TO PRINT THE RECORDED FRAME NUMBER AS VIDEO IS RECORDING??
}

Once the 5 second timer has elapsed the recording gets saved to camera roll. This works and I am able to review the captured video and it looks okay.

But never does func captureOutput get called. From my understanding the func captureOutput should be called each time a frame is added/processed while recoding video, or am I using an incorrect approach?

Here's my entire view controller:

import UIKit
import AVFoundation

class ViewController: UIViewController, AVCaptureFileOutputRecordingDelegate, AVCaptureVideoDataOutputSampleBufferDelegate {
    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        print("Record Frame Number: ") //HOW TO PRINT THE RECORDED FRAME NUMBER AS VIDEO IS RECORDING??
    }

    func fileOutput(_ output: AVCaptureFileOutput, didFinishRecordingTo outputFileURL: URL, from connections: [AVCaptureConnection], error: Error?) {
        print("FINISHED \(error)")
    }
    
    let videoOutput = AVCaptureVideoDataOutput();

    
    let captureSession : AVCaptureSession = AVCaptureSession()
    var captureDevice : AVCaptureDevice!
    var microphone : AVCaptureDevice!
    var previewLayer : AVCaptureVideoPreviewLayer!
    let videoFileOutput : AVCaptureMovieFileOutput = AVCaptureMovieFileOutput()
    var duration : Int = 5
    var v_path : URL = URL(fileURLWithPath: "")
    var my_timer : Timer = Timer()
    var cameraFront : Bool = false
    var cameras_number : Int = 0
    var max_zoom : CGFloat = 76
    var devices : [AVCaptureDevice] = []
    var captureInput : AVCaptureDeviceInput!
    var micInput : AVCaptureDeviceInput!
    let screenWidth = UIScreen.main.bounds.size.width

    override func viewDidLoad() {
        super.viewDidLoad()
        if (check_permissions())
        {
            initialize()
        }
        else
        {
            AVCaptureDevice.requestAccess(for: AVMediaType.video, completionHandler: { (granted) in
                if (granted)
                {
                    self.initialize()
                }
                else
                {
                    self.dismiss(animated: true, completion: nil)
                }
            })
        }
        takeVideoAction()
    }

    func configureDevice() {
        if let device = captureDevice {
            try! device.lockForConfiguration()
            device.focusMode = .locked
            device.unlockForConfiguration()
        }
    }
    
    func beginSession() {
        
        if (captureSession.isRunning)
        {
            captureSession.stopRunning()
        }
        do
        {
            try captureInput = AVCaptureDeviceInput(device: captureDevice)
            try micInput = AVCaptureDeviceInput(device: microphone)
            try captureDevice.lockForConfiguration()
        }
        catch
        {
            print(error)
        }
        // beginconfig before adding input and setting settings
        captureSession.beginConfiguration()
        captureSession.addInput(captureInput)
        captureSession.addInput(micInput)
        previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
        previewLayer.connection!.videoOrientation = AVCaptureVideoOrientation.init(rawValue: UIDevice.current.orientation.rawValue)!
        if (previewLayer.connection!.isVideoStabilizationSupported)
        {
            previewLayer.connection!.preferredVideoStabilizationMode = AVCaptureVideoStabilizationMode.auto
        }
        if (captureDevice.isSmoothAutoFocusSupported)
        {
            captureDevice.isSmoothAutoFocusEnabled = false
        }
        if (captureDevice.isFocusModeSupported(AVCaptureDevice.FocusMode.continuousAutoFocus))
        {
            captureDevice.focusMode = .continuousAutoFocus
        }
        set_preview_size_thing()
        set_quality_thing()
        if (captureDevice.isLowLightBoostSupported)
        {
            captureDevice.automaticallyEnablesLowLightBoostWhenAvailable = true
        }
        
        
        //NOT WORK
        if captureSession.canAddOutput(videoOutput) {
            captureSession.addOutput(videoOutput)
            
            videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "video_frames_queue", qos: .userInteractive, attributes: [], autoreleaseFrequency: .workItem))
            videoOutput.alwaysDiscardsLateVideoFrames = true
        }
        else {
            print("Input & Output could not be added to the session")
        }
               
        
        /*//NOT WORK
        let output = AVCaptureVideoDataOutput()
        output.alwaysDiscardsLateVideoFrames = true
        output.setSampleBufferDelegate(self, queue: sampleBufferQueue)
        
        captureSession.addOutput(output)
        */
        
        /*//NOT WORK
        let sampleBufferQueue = DispatchQueue.global(qos: .userInteractive)

        //var outputQueue : dispatch_queue_t?
        //outputQueue = dispatch_queue_create("outputQueue", DISPATCH_QUEUE_SERIAL);
        //output.setSampleBufferDelegate(self, queue: outputQueue)
        output.setSampleBufferDelegate(self, queue: sampleBufferQueue)
        output.alwaysDiscardsLateVideoFrames = true;
        output.videoSettings = nil;

        captureSession.addOutput(output);
        */
        
        captureSession.commitConfiguration()
        
        previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
        self.view.layer.addSublayer(previewLayer!)
        previewLayer?.frame = self.view.layer.frame
        
        captureSession.startRunning()
    }
    func initialize()
    {
        let directory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
        v_path = directory.appendingPathComponent("background.mp4")

        self.duration = 5
        devices = AVCaptureDevice.devices() as! [AVCaptureDevice]
        for device in devices
        {
            if (device.hasMediaType(AVMediaType.video))
            {
                if (device.position == AVCaptureDevice.Position.back)
                {
                    captureDevice = device as AVCaptureDevice
                }
                if (device.position == AVCaptureDevice.Position.front)
                {
                    cameras_number = 2
                }
            }
            if (device.hasMediaType(AVMediaType.audio))
            {
                microphone = device as AVCaptureDevice
            }
        }

        if captureDevice != nil
        {
            beginSession()
        }
        max_zoom = captureDevice.activeFormat.videoMaxZoomFactor
    }

    
    func takeVideoAction()
    {
    // movieFragmentInterval is important !! or you may end up with a video without audio
        videoFileOutput.movieFragmentInterval = CMTime.invalid
        captureSession.addOutput(videoFileOutput)
        (videoFileOutput.connections.first as! AVCaptureConnection).videoOrientation = returnedOrientation()
        videoFileOutput.maxRecordedDuration = CMTime(seconds: Double(self.duration), preferredTimescale: 1)
        videoFileOutput.startRecording(to: v_path, recordingDelegate: self)
    //timer will tell the remaining time
        my_timer = Timer.scheduledTimer(timeInterval: 1, target: self, selector: #selector(duration_thing), userInfo: nil, repeats: true)
    }
    
    func stopVideoAction()
    {
        print("STOPING")
        captureDevice.unlockForConfiguration()
        videoFileOutput.stopRecording()
        captureSession.stopRunning()
    // turn temp_video into an .mpeg4 (mp4) video
        let directory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
        let avAsset = AVURLAsset(url: v_path, options: nil)
    // there are other presets than AVAssetExportPresetPassthrough
        let exportSession = AVAssetExportSession(asset: avAsset, presetName: AVAssetExportPresetPassthrough)!
        exportSession.outputURL = directory.appendingPathComponent("main_video.mp4")
    // now it is actually in an mpeg4 container
        exportSession.outputFileType = AVFileType.mp4
        let start = CMTimeMakeWithSeconds(0.0, preferredTimescale: 0)
        let range = CMTimeRangeMake(start: start, duration: avAsset.duration)
        exportSession.timeRange = range
        exportSession.exportAsynchronously(completionHandler: {
            if (exportSession.status == AVAssetExportSession.Status.completed)
            {
        // you don’t need temp video after exporting main_video
                do
                {
                    try FileManager.default.removeItem(atPath: self.v_path.path)
                }
                catch
                {
                }
        // v_path is now points to mp4 main_video
                self.v_path = directory.appendingPathComponent("main_video.mp4")
                self.performSegue(withIdentifier: "ShareVideoController", sender: nil)
            }
        })
    }
    
    @objc func duration_thing()
    {
    // there is a textview to write remaining time left
        self.duration = self.duration - 1
        //timerTextView.text = "remaining seconds: \(self.duration)"
        //timerTextView.sizeToFit()
        if (self.duration == 0)
        {
            my_timer.invalidate()
            stopVideoAction()
        }
    }

    func returnedOrientation() -> AVCaptureVideoOrientation
    {
       var videoOrientation: AVCaptureVideoOrientation!
       let orientation = UIDevice.current.orientation
       switch orientation
       {
       case .landscapeLeft:
           videoOrientation = .landscapeRight
       case .landscapeRight:
           videoOrientation = .landscapeLeft
       default:
           videoOrientation = .landscapeLeft
       }
       return videoOrientation
    }

    
    func check_permissions() -> Bool
    {
        return AVCaptureDevice.authorizationStatus(for: AVMediaType.video) ==  AVAuthorizationStatus.authorized
    }


    func set_quality_thing()
    {
    // there is a switch in the screen (30-30 fps high quality or 15-23 fps normal quality)
    // you may not have to do this because export session also has some presets and a property called “optimizefornetwork” or something. But it would be better to make sure the output file is not huge with unnecessary 90 fps video
        captureDevice.activeVideoMinFrameDuration = CMTimeMake(value: 1, timescale: 30)
        captureDevice.activeVideoMaxFrameDuration = CMTimeMake(value: 1, timescale: 30)
    }

    func set_preview_size_thing()
    {
    //there is a switch for resolution (720p or 480p)
        captureSession.sessionPreset = AVCaptureSession.Preset.hd1280x720 //: AVCaptureSessionPreset640x480
    //this for loop is probably unnecessary and ridiculous but you can make sure you are using the right format
        for some_format in captureDevice.formats as! [AVCaptureDevice.Format]
        {
            let some_desc : String = String(describing: some_format)
            if (some_desc.contains("1280x") && some_desc.contains("720") && some_desc.contains("420v") && some_desc.contains("30 fps"))
            {
                captureDevice.activeFormat = some_format
                break
            }
        }
    }
}

I would like to be able to record video while simultaneously printing the current frame number of the recorded video.



Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source