Creating video from images with AVAssetWriter some frames are black

Question

I'm trying to make a video from individual photos using AVAssetWriter. My method of doing this is by constantly adding new pixel buffers to the same pixel buffer adaptor. Is this even the correct way of doing it? When I finish writing, some of the frames in the video are black.

Here's my current code:

import UIKit
import AVFoundation

struct MovieOutputSettings {
  let size: CGSize
  var fps: Int
  var avCodecKey = AVVideoCodecType.h264
  var videoFilename = "render"
  var videoFilenameExt = "mp4"
  var outputURL: URL
  
  init(size: CGSize = .zero, fps: Int = 1, avCodecKey: AVVideoCodecType = .h264, videoFilename: String = "render", videoFilenameExt: String = "mp4") {
    self.size = size
    self.fps = fps
    self.avCodecKey = avCodecKey
    self.videoFilename = videoFilename
    self.videoFilenameExt = videoFilenameExt
    self.outputURL = {
      let fileManager = FileManager.default
      if let tmpDirURL = try? fileManager.url(for: .cachesDirectory, in: .userDomainMask, appropriateFor: nil, create: true) {
        let url = tmpDirURL.appendingPathComponent(videoFilename).appendingPathExtension(videoFilenameExt)
        try? fileManager.removeItem(at: url)
        return url
      }
      fatalError("URLForDirectory() failed")
    }()
  }
}
 

class MovieMaker {
  let outputSettings: MovieOutputSettings
  let assetWriter: AVAssetWriter
  let input: AVAssetWriterInput
  let pixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor!
  var currentFrame = 0
  let timescale: Int32 = 600
  let lengthPerImage: Double
  
  init(outputSettings: MovieOutputSettings) {
    self.outputSettings = outputSettings
    lengthPerImage = 1.0 / Double(outputSettings.fps)
    print(lengthPerImage)
    do {
      self.assetWriter = try AVAssetWriter(outputURL: outputSettings.outputURL, fileType: .mp4)
      let outputSettings = [AVVideoCodecKey : AVVideoCodecType.h264, AVVideoWidthKey : NSNumber(floatLiteral: outputSettings.size.width), AVVideoHeightKey : NSNumber(floatLiteral: outputSettings.size.height)] as [String : Any]
      self.input = AVAssetWriterInput(mediaType: .video, outputSettings: outputSettings)
      self.assetWriter.add(input)
      print(assetWriter.canApply(outputSettings: AVOutputSettingsAssistant(preset: .hevc3840x2160WithAlpha)?.videoSettings!, forMediaType: .video))
    } catch {
      #warning("add error")
      fatalError()
    }
    pixelBufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: nil)
  }
  func buffer(from image: CIImage) -> CVPixelBuffer? {
    let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
    var pixelBuffer : CVPixelBuffer?
    let status = CVPixelBufferCreate(kCFAllocatorDefault, Int(image.extent.width), Int(image.extent.height), kCVPixelFormatType_32BGRA, attrs, &pixelBuffer)
    
    guard (status == kCVReturnSuccess) else {
      return nil
    }
    return pixelBuffer
  }
  func start() {
    assetWriter.startWriting()
    assetWriter.startSession(atSourceTime: .zero)
  }
  func addImage(image: UIImage) {
    let halfMovieLength = Float64(lengthPerImage / 2)
    let startFrameTime = CMTime(seconds: lengthPerImage * Double(currentFrame), preferredTimescale: timescale)
    let endFrameTime = CMTime(seconds: lengthPerImage * Double(currentFrame) + halfMovieLength, preferredTimescale: timescale)
    
    let ciImage = CIImage(cgImage: image.cgImage!)
    let pixelBuffer = buffer(from: ciImage)!
    print("is ready", pixelBufferAdaptor.assetWriterInput.isReadyForMoreMediaData)
    pixelBufferAdaptor.append(pixelBuffer, withPresentationTime: startFrameTime)
    pixelBufferAdaptor.append(pixelBuffer, withPresentationTime: endFrameTime)
    currentFrame += 1
  }
  func finish(completion: @escaping () -> ()) {
    input.markAsFinished()
    assetWriter.finishWriting {
      print(self.outputSettings.outputURL)
      completion()
    }
  }
}

Answer 1

I had to do something similar when developing PanoStory where I had to create a blank video by merging black images together and I used AVAssetWriter to create a video by appending pixel buffers of the black images to it.

I have to add that I am not an expert when it comes to working with CVPixelBuffer however, I can try to help you get your solution working.

Some things that I feel might be giving you the results you see:

1. The AVAssetWriterInputPixelBufferAdaptor is not properly configured
2. There is no setting of a start time / end time, you just need to keep track of the the frame duration and append pixel buffers accordingly
3. The conversion of a UIImage to a pixel buffer needs to be updated

Changes

I did not change much in MovieOutputSettings. I jut added var lengthPerImage: Double to it as it made more sense to me to have it in the output settings

Here are some changes I made to the MovieMaker struct

class MovieMaker {
    let outputSettings: MovieOutputSettings
    let assetWriter: AVAssetWriter
    let input: AVAssetWriterInput
    let pixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor!
    var currentFrame = 0
    let timescale: Int32 = 600
    
    // I added this to keep track of which image is being added
    // this helps when calculating the time
    var imageIndex = 0
    
    init(outputSettings: MovieOutputSettings) {
        self.outputSettings = outputSettings
        
        do {
            self.assetWriter = try AVAssetWriter(outputURL: outputSettings.outputURL, fileType: .mp4)
            
            let outputSettings = [AVVideoCodecKey : AVVideoCodecType.h264,
                                  AVVideoWidthKey : NSNumber(floatLiteral: Double(outputSettings.size.width)),
                                  AVVideoHeightKey : NSNumber(floatLiteral: Double(outputSettings.size.height))] as [String : Any]
            
            self.input = AVAssetWriterInput(mediaType: .video, outputSettings: outputSettings)
            self.assetWriter.add(input)
            print(assetWriter.canApply(outputSettings: AVOutputSettingsAssistant(preset: .hevc3840x2160WithAlpha)?.videoSettings!, forMediaType: .video))
        } catch {
            print("error: \(error)")
            fatalError()
        }
        
        // Set the required attributes for your pixel buffer adaptor
        let pixelBufferAttributes = [
            kCVPixelBufferPixelFormatTypeKey as String: NSNumber(value: kCVPixelFormatType_32ARGB),
            kCVPixelBufferWidthKey as String: NSNumber(value: Float(outputSettings.size.width)),
            kCVPixelBufferHeightKey as String: NSNumber(value: Float(outputSettings.size.height))
        ]
        
        pixelBufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input,
                                                                  sourcePixelBufferAttributes: pixelBufferAttributes)
    }
    
    // I updated this function of yours to buffer
    func pixelBufferFrom(_ image: UIImage,
                         pixelBufferPool: CVPixelBufferPool,
                         size: CGSize) -> CVPixelBuffer {
        
        var pixelBufferOut: CVPixelBuffer?
        
        let status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault,
                                                        pixelBufferPool,
                                                        &pixelBufferOut)
        
        if status != kCVReturnSuccess {
            fatalError("CVPixelBufferPoolCreatePixelBuffer() failed")
        }
        
        let pixelBuffer = pixelBufferOut!
        
        CVPixelBufferLockBaseAddress(pixelBuffer,
                                     CVPixelBufferLockFlags(rawValue: 0))
        
        let data = CVPixelBufferGetBaseAddress(pixelBuffer)
        let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
        
        let context = CGContext(data: data,
                                width: Int(size.width),
                                height: Int(size.height),
                                bitsPerComponent: 8,
                                bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer),
                                space: rgbColorSpace,
                                bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue)
        
        context?.clear(CGRect(x:0,
                              y: 0,
                              width: size.width,
                              height: size.height))
        
        let horizontalRatio = size.width / image.size.width
        let verticalRatio = size.height / image.size.height
        
        // ScaleAspectFit
        let aspectRatio = min(horizontalRatio,
                              verticalRatio)
        
        let newSize = CGSize(width: image.size.width * aspectRatio,
                             height: image.size.height * aspectRatio)
        
        let x = newSize.width < size.width ? (size.width - newSize.width) / 2 : 0
        let y = newSize.height < size.height ? (size.height - newSize.height) / 2 : 0
        
        context?.draw(image.cgImage!,
                      in: CGRect(x:x,
                                 y: y,
                                 width: newSize.width,
                                 height: newSize.height))
        
        CVPixelBufferUnlockBaseAddress(pixelBuffer,
                                       CVPixelBufferLockFlags(rawValue: 0))
        
        return pixelBuffer
    }
    
    func start() {
        assetWriter.startWriting()
        assetWriter.startSession(atSourceTime: .zero)
    }
    
    func addImage(image: UIImage) {
        // We don't set the start time and the end time but
        // rather the duration of each frame
        let frameDuration = CMTimeMake(value: Int64(timescale / Int32(outputSettings.fps)),
                                       timescale: timescale)
        
        imageIndex += 1
        
        // Keep adding the image for the required number of frames
        while currentFrame < imageIndex * (outputSettings.fps * Int(outputSettings.lengthPerImage))
        {
            // Convert the frame duration into the presentation time
            let presentationTime = CMTimeMultiply(frameDuration,
                                                  multiplier: Int32(currentFrame))
            
            let pixelBuffer = pixelBufferFrom(image,
                                              pixelBufferPool: pixelBufferAdaptor.pixelBufferPool!,
                                              size: outputSettings.size)
            
            print("is ready", pixelBufferAdaptor.assetWriterInput.isReadyForMoreMediaData)
            pixelBufferAdaptor.append(pixelBuffer,
                                      withPresentationTime: presentationTime)
            
            currentFrame += 1
        }
    }
    
    func finish(completion: @escaping (URL?) -> ()) {
        input.markAsFinished()
        
        // Reset the index
        imageIndex = 0
        
        assetWriter.finishWriting {
            print(self.outputSettings.outputURL)
            completion(self.outputSettings.outputURL)
        }
    }
}

And then I use it like this:

class ImageMovieVC: UIViewController
{
    let images = [UIImage(named: "art")!,
                  UIImage(named: "image-test")!,
                  UIImage(named: "dog")!,
                  UIImage(named: "art")!,
                  UIImage(named: "image-test")!,
                  UIImage(named: "dog")!]
    
    override func viewDidLoad()
    {
        super.viewDidLoad()
        view.backgroundColor = .white
        title = "Image Movie"
    }
    
    override func viewDidAppear(_ animated: Bool) {
        super.viewDidAppear(animated)
        
        // Video of size 1080x1920, 2 seconds for each image, 25fps
        let outputSettings = MovieOutputSettings(size: CGSize(width: 1080, height: 1920),
                                                 fps: 25,
                                                 lengthPerImage: 2.0)
        
        let movieMaker = MovieMaker(outputSettings: outputSettings)
        
        movieMaker.start()
        
        for image in images
        {
            movieMaker.addImage(image: image)
        }
        
        movieMaker.finish { [weak self] (url) in
            
            if let url = url
            {
                DispatchQueue.main.async {
                    self?.playVideo(url)
                }
            }
        }
    }
    
    // Function to play video player in AVPlayerViewController
    private func playVideo(_ url: URL)
    {
        let playerViewController = AVPlayerViewController()
        let player = AVPlayer(url: url)
        playerViewController.player = player
        
        present(playerViewController, animated: true) {
            playerViewController.player!.play()
        }
    }
}

And this gives me my images as a video with 2 seconds per image