ffmpeg command to create thumbnail sprites

Question

To output a single frame from ffmpeg I can do:

ffmpeg -i input.flv -ss 00:00:14.435 -vframes 1 out.png

And to output an image every second, I can do:

ffmpeg -i input.flv -vf fps=1 out%d.png

Would there be a way to create a thumbnail sprite from these outputs to help with the creation of a vtt file for thumbs on seek position?

Answer 1

Here is an example to create jpeg files (280x180) from mp4 video, using ffmpeg, and then assembling this thumbnails into a sprite (png format) using PHP gd2 + writing a VTT file for the video player.

First create 1 image per second with ffmpeg:

ffmpeg -i sprite/MyVideoFile.mp4 -r 1 -s 280x180 -f image2 sprite/thumbs/thumb-%%d.jpg

Then create sprite file + vtt file (example with PHP):

$dirToScan      =   'thumbs/';
$filePrefix     =   'thumb-';
$fileSuffix     =   '.jpg';
$thumbWidth     =   280;
$thumbHeight    =   180;
$imageFiles     =   array();
$spriteFile     =   'sprite.png';
$imageLine      =   20;
$vttFile        =   'sprite.vtt';
$dst_x          =   0;
$dst_y          =   0;


# read the directory with thumbnails, file name in array
foreach (glob($dirToScan.$filePrefix.'*'.$fileSuffix) as $filename) {
    array_push($imageFiles,$filename);
}

natsort($imageFiles);
#calculate dimension for the sprite 
        $spriteWidth =  $thumbWidth*$imageLine;
        $spriteHeight   =   $thumbHeight*(floor(count($imageFiles)/$imageLine)+1);

        # create png file for sprite
        $png = imagecreatetruecolor($spriteWidth,$spriteHeight);

        # open vtt file
            $handle =   fopen($vttFile,'wb+');
            fwrite($handle,'WEBVTT'."\n");

        # insert thumbs in sprite and write the vtt file
            foreach($imageFiles AS $file)   {
                $counter        =   str_replace($filePrefix,'',str_replace($fileSuffix,'',str_replace($dirToScan,'',$file)));
                $imageSrc = imagecreatefromjpeg($file);
                imagecopyresized ($png, $imageSrc, $dst_x , $dst_y , 0, 0, $thumbWidth, $thumbHeight, $thumbWidth,$thumbHeight);

                $varTCstart =   gmdate("H:i:s", $counter-1).'.000';
                $varTCend   =   gmdate("H:i:s", $counter).'.000';

                $varSprite  =   $spriteFile.'#xywh='.$dst_x.','.$dst_y.','.$thumbWidth.','.$thumbHeight;

                fwrite($handle,$counter."\n".$varTCstart.' --> '.$varTCend."\n".$varSprite."\n\n");

create new line after 20 images
                if ($counter % $imageLine == 0) {
                    $dst_x=0;
                    $dst_y+=$thumbHeight;
                }
                else    {
                    $dst_x+=$thumbWidth;
                }

            }
            imagepng($png,$spriteFile);
            fclose($handle);

VTT file looks like:

WEBVTT
1
00:00:00.000 --> 00:00:01.000
sprite.png#xywh=0,0,280,180

2
00:00:01.000 --> 00:00:02.000
sprite.png#xywh=280,0,280,180

3
00:00:02.000 --> 00:00:03.000
sprite.png#xywh=560,0,280,180
...

Answer 2

A little improvement proposal to the answer given by MR_1204: if the final sprite is a PNG, I would avoid the quality decrease caused by JPG compression and rather save the temporary thumbnails as PNGs:

ffmpeg -i sprite/MyVideoFile.mp4 -r 1 -s 280x180 -f image2 sprite/thumbs/thumb-%%d.png

Also, saving to PNG is usually a (tiny) little bit faster than saving to JPG.

Instead, to keep the download small, it might make sense to save (only) the final sprite image as JPG, which in case of PHP and GD only requires to replace imagepng(...) with imagejpeg(...).

Answer 3

Python solution of MR_1204's solution

import ffmpeg
import logging
from pathlib import Path
from config import temp
import os
from PIL import Image
from datetime import datetime, timedelta
import math
logger = logging.getLogger(__name__)


class ScreenshotExtractor:
    def __init__(self):
        self.screenshot_folder = f"{temp}ss"
        self.gap = 10
        self.size = "177x100"
        self.col = 5
        self.ss_width = 177
        self.ss_height = 100

    def create_sprite_sheet(self, name):
        path, dirs, files = next(os.walk(self.screenshot_folder))
        file_count = len(files)
        img_width = self.ss_width * self.col
        img_height = self.ss_height * math.ceil(file_count/self.col)

        file_name = f"{name}.jpg"
        out_image = Image.new('RGB', (img_width, img_height))
        webvtt = "WEBVTT\n\n"
        for count, img_file in enumerate(files):
            img = Image.open(f"{self.screenshot_folder}/{img_file}")
            st_time = timedelta(seconds=count * self.gap)
            end_time = timedelta(seconds=(count + 1) * self.gap)

            # Adding img to out_file
            x_mod = int(count / self.col)
            x = 0 + (count - (self.col * x_mod)) * self.ss_width
            y = 0 + x_mod * self.ss_height
            out_image.paste(img, (x, y))

            sprite = f"{file_name}#xywh={x},{y},{self.ss_width},{self.ss_height}"
            webvtt += f"{count + 1}\n0{str(st_time)}.000 --> 0{str(end_time)}.000\n{sprite}\n\n"

        out_image.save(f"{temp}{file_name}", quality=90)
        f = open(f"{temp}{name}.vtt", "w")
        f.write(webvtt)
        f.close()
        return True

    def extract_screenshots(self, file_uri, name):
        try:
            Path(self.screenshot_folder).mkdir(parents=True, exist_ok=True)

            vod = ffmpeg.input(file_uri)
            vod.output(f"{self.screenshot_folder}/{name}_%04d.png", r=1/self.gap, s=self.size).run()
            return True
        except Exception as e:
            logger.warning(e)
            return False

    def run(self, file_uri, name):
        # TODO - Actually do logic here
        self.extract_screenshots(file_uri, name)
        self.create_sprite_sheet(name)

Hope this helps somebody