GStreamer timestamps (PTS) are not monotonically increasing for captured frames

Question

I wrote some code to capture frames from a webcam using GStreamer 1.0 (PyGST). It is important for me to know the exact time of capture. For this, I set the v4l2src property do-timestamp, and I use appsink to write the buffer PTS to a text file.

However, the timestamps are not monotonically increasing. E.g. the timestamp of frame 16 is 0.88199 s, and the timstamp of frame 17 is 0.77462 s, i.e. 0.10737 s EARLIER than previous frame. (I have a figure showing the problem, but lack the reputation necessary to post it.)

Is it correct that PTS of captured GstBuffers are not always monotonically increasing? If this is not normal behavior, does anyone know what I messed up?

I use a Logitech c920 webcam. The frames are h.264 encoded on the camera. The code looks roughly like this:

import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, Gtk
GObject.threads_init()
Gst.init(None)

class Webcam:
def __init__(self, video_dev='/dev/video0', fps=30):

    ts_log_fname = 'webcam_timestamps.log'
    vid_fname = 'webcam.mkv'
    self.ts_log = open(ts_log_fname, 'w')
    self.ts_log.write('video filename: %s\n '
                      '\nframe_number, cam_running_ts\n' % vid_fname)

    self.n_frames = 0

    # Create GStreamer pipline
    self.pipeline = Gst.Pipeline()

    # Create bus to get events from GStreamer pipeline
    self.bus = self.pipeline.get_bus()
    self.bus.add_signal_watch()
    self.bus.connect('message::error', self.on_error)
    self.bus.enable_sync_message_emission()
    self.bus.connect('sync-message::element', self.on_sync_message)

    ###########################
    # Callable function
    ###########################
    def on_new_sample(appsink):
        """
        Function called from the pipeline by appsink.
        Writes the timestampes of frame capture to a log file.
        """
        # Get the buffer
        smp = appsink.emit('pull-sample')
        buf = smp.get_buffer()
        # Nanoseconds to seconds
        timestamp = np.float64(1e-9) * buf.pts
        self.n_frames += 1
        self.ts_log.write('%d,%0.9f\n' % (self.n_frames, timestamp))
        return False

    ###########################
    # Create GStreamer elements
    ###########################
    # Video source:
    self.v4l2src0 = Gst.ElementFactory.make('v4l2src', None)
    self.v4l2src0.set_property('device', video_dev)
    self.v4l2src0.set_property('do-timestamp', 'true')
    # Video source filters:
    vid0caps = Gst.Caps.from_string('video/x-h264,width=%d,height=%d,'
                                    'framerate=%d/1' % (1280, 720, fps))
    self.vid0filter = Gst.ElementFactory.make('capsfilter', None)
    self.vid0filter.set_property('caps', vid0caps)
    # Parse video:
    self.vid0parse = Gst.ElementFactory.make('h264parse', None)
    # Split:
    self.tee0 = Gst.ElementFactory.make('tee', None)
    self.tee0.set_property('name', 't0')
    ####
    # Display branch
    ####
    # Decode
    self.vid0decode = Gst.ElementFactory.make('avdec_h264', None)
    # Scale to display size:
    self.disp0scale = Gst.ElementFactory.make('videoscale', None)
    # Display filter caps:
    disp0caps = Gst.Caps.from_string('video/x-raw,width=%d,height=%d' %
                                     (800, 600))
    # Sinks:
    self.disp0sink = Gst.ElementFactory.make('autovideosink', None)
    self.disp0sink.set_property('filter-caps', disp0caps)
    ####
    # File branch
    ####
    self.mux = Gst.ElementFactory.make('matroskamux', None)
    self.file0sink = Gst.ElementFactory.make('filesink', None)
    self.file0sink.set_property('location', vid_fname)
    self.file0sink.set_property('sync', False)
    ####
    # Timestamp branch
    ####
    # Create appsink
    self.ts0sink = Gst.ElementFactory.make('appsink', None)
    # Setting properties of appsink
    ts0caps = vid0caps  # use same caps as for camera
    self.ts0sink.set_property('caps', ts0caps)
    self.ts0sink.set_property("max-buffers", 20)  # Limit memory usage
    # Tell sink to emit signals
    self.ts0sink.set_property('emit-signals', True)
    self.ts0sink.set_property('sync', False)  # No sync
    # Connect appsink to my function (writing timestamps)
    self.ts0sink.connect('new-sample', on_new_sample)

    self.queue0 = Gst.ElementFactory.make('queue', None)
    self.queue1 = Gst.ElementFactory.make('queue', None)                
    self.disp_queue = Gst.ElementFactory.make('queue', None)
    self.file_queue = Gst.ElementFactory.make('queue', None)
    self.ts_queue = Gst.ElementFactory.make('queue', None)

    # Add elements to the pipeline
    self.pipeline.add(self.v4l2src0)
    self.pipeline.add(self.vid0filter)
    self.pipeline.add(self.vid0parse)
    self.pipeline.add(self.tee0)
    self.pipeline.add(self.vid0decode)
    self.pipeline.add(self.disp0scale)
    self.pipeline.add(self.disp0sink)
    self.pipeline.add(self.mux)
    self.pipeline.add(self.file0sink)
    self.pipeline.add(self.ts0sink)
    self.pipeline.add(self.queue0)
    self.pipeline.add(self.queue1)
    self.pipeline.add(self.disp_queue)
    self.pipeline.add(self.file_queue)
    self.pipeline.add(self.ts_queue)

    ###############
    # Link elements
    ###############
    # video source
    if not self.v4l2src0.link(self.vid0filter):
        print('video source to video filter link failed')          
    if not self.vid0filter.link(self.vid0parse):
        print('video filter to video parse link failed')
    if not self.vid0parse.link(self.tee0):
        print('video parse to tee link failed')    
    # tee
    if not self.tee0.link(self.disp_queue):
        print('tee to display queue link failed')
    if not self.tee0.link(self.file_queue):
        print('tee to file queue link failed')
    if not self.tee0.link(self.ts_queue):
        print('tee to ts queue link failed')
    # video display sink
    if not self.disp_queue.link(self.vid0decode):
        print('dispaly queue to video decode link failed')
    if not self.vid0decode.link(self.disp0scale):
        print('decode to videoscale link failed')
    if not self.disp0scale.link(self.queue0):
        print('disp0scale to queue0 link failed')            
    if not self.queue0.link_filtered(self.disp0sink, disp0caps):
        print('queue0 to display-sink link failed')
    # file sink
    if not self.file_queue.link(self.mux):
        print('file queue to mux link failed')           
    if not self.mux.link(self.queue1):
        print('mux to queue1 link failed')            
    if not self.queue1.link(self.file0sink):
        print('queue1 to file-sink link failed')
    # timestamp sink
    if not self.ts_queue.link(self.ts0sink):
        print('ts queue to ts-sink link failed')

def run(self):
    self.offset_t = datetime.now().timestamp() - self.t_start
    self.pipeline.set_state(Gst.State.PLAYING)

def quit(self):
    self.pipeline.set_state(Gst.State.NULL)
    self.ts_log.close()

def on_sync_message(self, bus, msg):
    if msg.get_structure().get_name() == 'prepare-window-handle':
        msg.src.set_property('force-aspect-ratio', True)

def on_error(self, bus, msg):
    print('on_error():', msg.parse_error())

Answer 1

As suggested by mpr, the non-monotonic timestamps occur when streaming h.264. It also seems to occur just in the beginning of the stream. The problem disappears when streaming raw video.

The big, 300 ms, jumps every 30 frames or so seems to be something specific to the Logitech C920 camera. When I use an onboard laptop camera, the jumps are smaller, 130 ms, and fewer, every 85 frames or so.