How to record depth stream from realsense L515

Question

I have a depth camera (Intel Realsense L515) and I do like to record a video of the depth.

I have seen this answer which is using FFMPEG, but I didn't know how to replicate it in my case!

I'm using this code:

import cv2
import numpy as np
import pyrealsense2 as rs
import time

pipeline = rs.pipeline()
config = rs.config()

"""
# Depth Mode
"""
# Resolution
res = [(1024, 768), (640, 480), (320, 240)]
resolution = res[0]
print("RealSense Resolution:{}\n".format(resolution))

# # initialize video writer
fourcc = cv2.VideoWriter_fourcc('F','F','V','1')
fps = 30
video_filename = 'output.avi'
out = cv2.VideoWriter(video_filename, fourcc, fps, resolution, False)


config.enable_stream(rs.stream.depth, resolution[0], resolution[1], rs.format.z16, 30)
profile = config.resolve(pipeline)
# Start streaming
pipeline.start(config)

# Declare sensor object and set options
depth_sensor = profile.get_device().first_depth_sensor()
depth_sensor.set_option(rs.option.visual_preset, 5) # 5 is short range, 3 is low ambient light
depth_sensor.set_option(rs.option.receiver_gain, 8)
depth_sensor.set_option(rs.option.pre_processing_sharpening, 0.0)
depth_sensor.set_option(rs.option.post_processing_sharpening, 3.0)
depth_sensor.set_option(rs.option.laser_power, 100)
depth_sensor.set_option(rs.option.confidence_threshold, 2)
# Get the sensor once at the beginning. (Sensor index: 1)

# # Filters
threshold_filter = rs.threshold_filter(min_dist=1.2, max_dist=1.4)
temporal_filter = rs.temporal_filter(smooth_alpha=0.1, smooth_delta = 9.0,persistence_control=7)

try:
    # # Filters
    threshold_filter = rs.threshold_filter(min_dist=1.2, max_dist=1.4)
    temporal_filter = rs.temporal_filter(smooth_alpha=0.1, smooth_delta = 75.0,persistence_control=0)

    tic = time.time()

    while True:
        # Wait for depth frames:
        frames = pipeline.wait_for_frames()
        depth_frame = frames.get_depth_frame()
        if not depth_frame:
            continue

        #------------
        # # FILTERS |
        #------------

        depth_frame = threshold_filter.process(depth_frame)
        depth_frame = temporal_filter.process(depth_frame)

        # Convert images to numpy arrays
        depth_array = np.asanyarray(depth_frame.get_data())
        # depth_array = np.asanyarray(colorizer.colorize(depth_frame).get_data())

        out.write(depth_array)
        toc = time.time()
        if(round(toc  - tic) > 30):
            break

finally:
    out.release()
    pipeline.stop()

And getting this error:

out.write(depth_array) cv2.error: OpenCV(4.5.4) /tmp/pip-req-build-kneyjnox/opencv/modules/videoio/src/cap_ffmpeg.cpp:186: error: (-215:Assertion failed) image.depth() == CV_8U in function 'write'

Can you please tell me how can I record the depth from my camera? thanks in advance.

Answer 1

I don't know if you can specify pix_fmt with the opencv's ffmpeg submodule, but here is an alternative using my ffmpegio package (GitHub)

pip install ffmpegio

import ffmpegio
import numpy as np

fps = 30
video_filename = "sandbox/output.avi"
shape = (640, 480)
with ffmpegio.open(
    video_filename,
    "wv", # mode of operation: video write 
    fps,
    c="ffv1", # c is sufficient for video-only, o.w. `**{'c:v': 'ffv1'}`
    overwrite=True, # False (or remove) to error out if output file exists
    show_log=True, # False (or remove) to stop showing FFmpeg log on console
) as out:

    # ...

    while True:

        # ...

        out.write(depth_array) # first write will configure video format:
        # dtype=np.uint16 -> pix_fmt='gray16le'
        # shape=(h,w) -> s=f'{w}x{h}

        # ...

Note: ffmpegio requires ffmpeg binary to be available on the system and discoverable (unlike opencv).

You can also achieve the same with plain python subprocess with additional info up front:

import subprocess as sp

ffmpeg_path = 'ffmpeg'

fps = 30
video_filename = 'output.avi'
width = 640
height = 480

out = sp.Popen([ffmpeg_path, '-y',
               '-f','rawvideo','-r',str(fps),'-pix_fmt','gray16le','-s',f'{width}x{height}','-',
               '-f','avi','-c:v','ffv1',video_filename],stdin=sp.PIPE):

try:
    ...
    while True:
        ...
        depth_array = np.asanyarray(depth_frame.get_data())

        out.stdin.write(depth_array) # you may need to wrap with memory

        ...

finally:
    out.stdin.close()
    out.wait()
    ...