Expose probe score, codec frame rate, and frame cropping

CHANGELOG.rst

@@ -34,6 +34,9 @@ v19.1.0 (Unreleased)
 Features:
 
 - Add ``VideoFrame.solid()`` to create a frame filled with one color, in any non-paletted pixel format.
+- Add ``InputContainer.probe_score``, ``InputContainer.duration_estimation_method``, ``VideoStream.codec_rate``, ``VideoStream.video_delay``, ``Codec.wrapper_name``, and ``Codec.max_lowres``.
+- Add ``VideoFrame.crop_top``, ``crop_bottom``, ``crop_left``, ``crop_right``, and ``VideoFrame.apply_cropping()``.
+- Add ``Frame.decode_error_flags`` as a ``DecodeError`` flag and ``VideoFrame.repeat_pict``.
 - Add ``Stream.cctx`` as the name for a stream's codec context; ``Stream.codec_context`` remains as an alias.
 - Deprecate ``AudioStream.format``, ``AudioStream.layout``, ``VideoStream.format``, ``VideoStream.thread_count``, and ``VideoStream.thread_type``, which pass through to the codec context; use ``stream.cctx.format``, ``stream.cctx.layout``, ``stream.cctx.thread_count``, and ``stream.cctx.thread_type`` instead. Passing these to ``add_stream()`` is not deprecated.
 

av/codec/codec.py

@@ -178,6 +178,35 @@ def canonical_name(self):
     def long_name(self):
         return self.ptr.long_name or ""
 
+    @property
+    def wrapper_name(self):
+        """
+        The name of the external library or hardware API this codec wraps, or
+        ``None`` for codecs native to FFmpeg.
+
+        E.g: ``'at'`` for ``aac_at``, which wraps AudioToolbox. Not every
+        wrapper sets this; FFmpeg 9.0 added it to more of them, such as
+        ``h264_videotoolbox`` and ``libx264``.
+
+        Wraps :ffmpeg:`AVCodec.wrapper_name`.
+
+        :rtype: str | None
+        """
+        if self.ptr.wrapper_name != cython.NULL:
+            return self.ptr.wrapper_name
+
+    @property
+    def max_lowres(self):
+        """
+        The highest ``lowres`` option value this decoder supports, or ``0`` if it
+        cannot decode at reduced resolution.
+
+        Wraps :ffmpeg:`AVCodec.max_lowres`.
+
+        :rtype: int
+        """
+        return self.ptr.max_lowres
+
     @property
     def type(self):
         """

av/codec/codec.pyi

@@ -72,6 +72,10 @@ class Codec:
     @property
     def long_name(self) -> str: ...
     @property
+    def wrapper_name(self) -> str | None: ...
+    @property
+    def max_lowres(self) -> int: ...
+    @property
     def type(
         self,
     ) -> Literal["video", "audio", "data", "subtitle", "attachment", "unknown"]: ...

av/container/input.py

@@ -145,6 +145,43 @@ def size(self):
         if size >= 0:
             return size
 
+    @property
+    def probe_score(self):
+        """How confident FFmpeg was when it detected the input format, from 0 to 100.
+
+        Low scores mean the format was a guess. It is 0 when ``format=`` was
+        given, since no probing happens.
+
+        Wraps :ffmpeg:`AVFormatContext.probe_score`.
+
+        :type: int
+        """
+        self._assert_open()
+        return self.ptr.probe_score
+
+    @property
+    def duration_estimation_method(self):
+        """How FFmpeg worked out :attr:`duration`, or ``None`` if it is unknown.
+
+        - ``"pts"``: from the timestamps of the packets, which is accurate.
+        - ``"stream"``: from a stream with a known duration.
+        - ``"bitrate"``: from the file size and bitrate, which can be far off,
+          especially for variable bitrate files.
+
+        Wraps :ffmpeg:`AVFormatContext.duration_estimation_method`.
+
+        :type: Literal["pts", "stream", "bitrate"] | None
+        """
+        self._assert_open()
+        if self.ptr.duration == lib.AV_NOPTS_VALUE:
+            return None
+        method = self.ptr.duration_estimation_method
+        if method == lib.AVFMT_DURATION_FROM_PTS:
+            return "pts"
+        if method == lib.AVFMT_DURATION_FROM_STREAM:
+            return "stream"
+        return "bitrate"
+
     def close(self):
         close_input(self)
 

av/container/input.pyi

@@ -1,5 +1,5 @@
 from collections.abc import Iterator
-from typing import final, overload
+from typing import Literal, final, overload
 
 from av.audio.frame import AudioFrame
 from av.audio.stream import AudioStream
@@ -20,6 +20,8 @@ class InputContainer(Container):
     duration: int | None
     bit_rate: int
     size: int | None
+    probe_score: int
+    duration_estimation_method: Literal["pts", "stream", "bitrate"] | None
 
     @overload
     def demux(self, video_stream: VideoStream) -> Iterator[Packet[VideoStream]]: ...

av/frame.py

@@ -1,3 +1,5 @@
+from enum import IntFlag
+
 import cython
 from cython.cimports.av.error import err_check
 from cython.cimports.av.opaque import opaque_container
@@ -10,6 +12,13 @@
 from av.sidedata.sidedata import SideDataContainer
 
 
+class DecodeError(IntFlag):
+    invalid_bitstream = lib.FF_DECODE_ERROR_INVALID_BITSTREAM
+    missing_reference = lib.FF_DECODE_ERROR_MISSING_REFERENCE
+    concealment_active = lib.FF_DECODE_ERROR_CONCEALMENT_ACTIVE
+    decode_slices = lib.FF_DECODE_ERROR_DECODE_SLICES
+
+
 @cython.cclass
 class Frame:
     """
@@ -186,6 +195,19 @@ def is_corrupt(self):
             self.ptr.flags & lib.AV_FRAME_FLAG_CORRUPT
         )
 
+    @property
+    def decode_error_flags(self):
+        """What went wrong while decoding this frame, if anything.
+
+        The decoder still produced the frame, but it may contain artifacts.
+        Empty for frames that decoded cleanly.
+
+        Wraps :ffmpeg:`AVFrame.decode_error_flags`.
+
+        :type: DecodeError
+        """
+        return DecodeError(self.ptr.decode_error_flags)
+
     @property
     def key_frame(self):
         """Is this frame a key frame?

av/frame.pyi

@@ -1,10 +1,17 @@
+from enum import IntFlag
 from fractions import Fraction
 
 from typing_extensions import disjoint_base
 
 from av.rational import AVRational
 from av.sidedata.sidedata import SideDataContainer
 
+class DecodeError(IntFlag):
+    invalid_bitstream = ...
+    missing_reference = ...
+    concealment_active = ...
+    decode_slices = ...
+
 @disjoint_base
 class Frame:
     dts: int | None
@@ -26,5 +33,7 @@ class Frame:
     @property
     def is_corrupt(self) -> bool: ...
     @property
+    def decode_error_flags(self) -> DecodeError: ...
+    @property
     def key_frame(self) -> bool: ...
     def make_writable(self) -> None: ...

av/video/frame.py

@@ -714,6 +714,82 @@ def interlaced_frame(self):
 
         return bool(self.ptr.flags & lib.AV_FRAME_FLAG_INTERLACED)
 
+    @property
+    def crop_top(self):
+        """Pixels to crop from the top of the frame.
+
+        Decoders apply cropping themselves unless the ``apply_cropping`` codec
+        option is ``"0"``, in which case the crop is reported here and can be
+        applied with :meth:`apply_cropping`.
+
+        Wraps :ffmpeg:`AVFrame.crop_top`.
+
+        :type: int
+        """
+        return self.ptr.crop_top
+
+    @crop_top.setter
+    def crop_top(self, value):
+        self.ptr.crop_top = value
+
+    @property
+    def crop_bottom(self):
+        """Pixels to crop from the bottom of the frame. See :attr:`crop_top`.
+
+        Wraps :ffmpeg:`AVFrame.crop_bottom`.
+
+        :type: int
+        """
+        return self.ptr.crop_bottom
+
+    @crop_bottom.setter
+    def crop_bottom(self, value):
+        self.ptr.crop_bottom = value
+
+    @property
+    def crop_left(self):
+        """Pixels to crop from the left of the frame. See :attr:`crop_top`.
+
+        Wraps :ffmpeg:`AVFrame.crop_left`.
+
+        :type: int
+        """
+        return self.ptr.crop_left
+
+    @crop_left.setter
+    def crop_left(self, value):
+        self.ptr.crop_left = value
+
+    @property
+    def crop_right(self):
+        """Pixels to crop from the right of the frame. See :attr:`crop_top`.
+
+        Wraps :ffmpeg:`AVFrame.crop_right`.
+
+        :type: int
+        """
+        return self.ptr.crop_right
+
+    @crop_right.setter
+    def crop_right(self, value):
+        self.ptr.crop_right = value
+
+    def apply_cropping(self, unaligned=False):
+        """Crop the frame in place according to its ``crop_*`` attributes.
+
+        No pixels are copied; the planes are narrowed to the cropped region and
+        the ``crop_*`` attributes are reset to 0.
+
+        By default the left edge may be moved slightly to keep plane data
+        aligned, leaving a few extra pixels. Pass ``unaligned=True`` to crop
+        exactly.
+
+        Wraps :ffmpeg:`av_frame_apply_cropping`.
+        """
+        flags: cython.int = lib.AV_FRAME_CROP_UNALIGNED if unaligned else 0
+        err_check(lib.av_frame_apply_cropping(self.ptr, flags))
+        self._init_user_attributes()
+
     @property
     def pict_type(self):
         """Returns an integer that corresponds to the PictureType enum.
@@ -728,6 +804,26 @@ def pict_type(self):
     def pict_type(self, value):
         self.ptr.pict_type = value
 
+    @property
+    def repeat_pict(self):
+        """How many extra fields to show this frame for, as used by soft
+        telecine.
+
+        The frame lasts ``repeat_pict + 2`` field durations, so ``1`` on an
+        interlaced frame means it is shown as 3 fields. Decoders set this from
+        MPEG-2 ``repeat_first_field`` or H.264 ``pic_struct``. Few encoders
+        read it; Intel QSV is one.
+
+        Wraps :ffmpeg:`AVFrame.repeat_pict`.
+
+        :type: int
+        """
+        return self.ptr.repeat_pict
+
+    @repeat_pict.setter
+    def repeat_pict(self, value):
+        self.ptr.repeat_pict = value
+
     @property
     def colorspace(self):
         """The YUV/RGB transformation matrix of the frame, as FFmpeg's raw

av/video/frame.pyi

@@ -69,6 +69,7 @@ class VideoFrame(Frame):
     format: VideoFormat
     planes: tuple[VideoPlane, ...]
     pict_type: int
+    repeat_pict: int
     colorspace: int
     color_range: int
     color_trc: int
@@ -83,6 +84,11 @@ class VideoFrame(Frame):
     def height(self) -> int: ...
     @property
     def interlaced_frame(self) -> bool: ...
+    crop_top: int
+    crop_bottom: int
+    crop_left: int
+    crop_right: int
+    def apply_cropping(self, unaligned: bool = False) -> None: ...
     @property
     def rotation(self) -> int: ...
     def __new__(

av/video/plane.py

@@ -51,6 +51,22 @@ def __cinit__(self, frame: VideoFrame, index: cython.int):
         # ignore it's direction.
         self.buffer_size = abs(self.frame.ptr.linesize[self.index]) * self.height
 
+        # Cropping moves the data pointer into the buffer, so the last row can end
+        # past the allocation. Clamp to what the plane's buffer actually holds.
+        if frame.ptr.hw_frames_ctx or self.frame.ptr.linesize[self.index] < 0:
+            return
+        buf: cython.pointer[lib.AVBufferRef] = lib.av_frame_get_plane_buffer(
+            frame.ptr, index
+        )
+        if buf == cython.NULL:
+            return
+        start: cython.p_uchar = frame.ptr.data[index]
+        end: cython.p_uchar = buf.data + buf.size
+        if buf.data <= start < end and cython.cast(cython.size_t, end - start) < (
+            self.buffer_size
+        ):
+            self.buffer_size = end - start
+
     @cython.cfunc
     def _buffer_size(self) -> cython.size_t:
         return self.buffer_size

av/video/stream.py

@@ -259,6 +259,32 @@ def guessed_rate(self):
         )
         return from_avrational(val)
 
+    @property
+    def codec_rate(self):
+        """The frame rate stored in the codec bitstream, such as H.264 VUI
+        timing info.
+
+        Is falsy if the bitstream does not say.
+
+        Wraps :ffmpeg:`AVCodecParameters.framerate`.
+
+        :type: AVRational
+        """
+        self._assert_open()
+        return from_avrational(self.ptr.codecpar.framerate)
+
+    @property
+    def video_delay(self):
+        """The number of frames the decoder holds back before output, due to
+        frame reordering.
+
+        Wraps :ffmpeg:`AVCodecParameters.video_delay`.
+
+        :type: int
+        """
+        self._assert_open()
+        return self.ptr.codecpar.video_delay
+
     @property
     def sample_aspect_ratio(self):
         """The guessed sample aspect ratio (SAR) of this stream.

av/video/stream.pyi

@@ -36,6 +36,8 @@ class VideoStream(Stream):
     average_rate: AVRational
     base_rate: AVRational
     guessed_rate: AVRational
+    codec_rate: AVRational
+    video_delay: int
 
     # from codec context
     name: str