vidi/pipelines/

mod.rs

/*
 * SPDX-FileCopyrightText: 2023 Purism, SPC <https://puri.sm>
 *
 * SPDX-License-Identifier: MPL-2.0 OR LGPL-2.1-or-later
 */

/*! General V4L2 handling.

This module should take over a lot of things from the uvc module because UVC should only be specialized to do things that aren't applying to every camera.

It's still unclear what the interface to a specialized module should be. Some drivers use special formats for exchanging statistics and creating data to feed back to the hardware with them. Those formats are well-known and described at runtime, so perhaps they can be just a different pipeline config rather than an entirely new driver.
*/

mod uvc;

use crate::actors::camera_list::CreationKit;
use crate::actors::watcher_udev as udev;
use crate::config::{Config, ConfigsDatabase, DeviceConfig, PadState, PipelineState};
use crate::io::media;
use crate::io::subdev::{self, MbusFrameFormat};
use crate::search;
use crate::search::{Database, TopologyDatabase};
use crate::storage::{FileConfig, Io};
use error_backtrace::{IntoTraced, Result as TracedResult, ResultBacktrace};
use media_subsystem::{EntityName, MediaV2Entity, MediaLinkDesc, MediaPadDesc, LinkEnabled};
use parking_lot::{ArcMutexGuard, Mutex, RawMutex};
use v4l2_subdev::MbusFrameFormatFlags;
use std::borrow::BorrowMut;
use std::error;
use std::fmt;
use std::io;
use std::ops::{Deref, DerefMut};
use std::sync::Arc;
use thiserror;
use tracing::{trace, warn};
use v4l::{Control, Device};
use v4l::buffer;
use v4l::control;
use v4l::capability;
use v4l::framesize::FrameSize;
use v4l::io::dmabuf;
use v4l::video::Capture;

pub type Error = Box<dyn error::Error>;

/// An ID of a camera, unique between cameras on the same system, same across instances of this camera. Undefined across library versions or consecutive runs.
#[derive(Eq, PartialEq, Clone, Default, Debug)]
pub struct CameraHash(u64);

// Instantiating an arbitrary hash must not be public. Otherwise, the user could assign an untested config to any camera.

impl From<&CameraId> for CameraHash {
    fn from(value: &CameraId) -> Self {
        use std::hash;
        use std::hash::{Hash, Hasher};
        let mut s = hash::DefaultHasher::new();
        value.hash(&mut s);
        CameraHash(s.finish())
    }
}

pub type CameraId = String;

/// Information about present camera
/// The camera is defined as a sensor which can reach an output, without specifying any path between them. The camera may fail to get acquired if all outputs are already acquired by other cameras.
#[derive(Clone, Debug)]
pub struct CameraInfo {
    /// Information about the media device
    device: udev::Device,
    /// ID assigned by this library (and the relevant backend)
    id: CameraId,
    /// The sensor defining this camera
    sensor: EntityName,
}

impl CameraInfo {
    pub fn id(&self) -> &CameraId {
        &self.id
    }
    pub fn is_for_device(&self, d: &udev::Device) -> bool {
        self.device == *d
    }
}

fn apply_video_config(config: &Config, mut format: v4l::Format) -> v4l::Format {
    format.fourcc = config.fourcc;
    format.width = config.width;
    format.height = config.height;
    format
}

#[derive(thiserror::Error, Debug)]
pub enum AcquireError {
    #[error("Camera already acquired elsewhere")]
    AlreadyAcquired,
    #[error("Other error")]
    Other(())
}

pub type StreamBorrowing = Stream<dmabuf::Stream>;
pub type StreamManual = Stream<dmabuf::StreamManual>;

/// A way to stream buffers
pub struct Stream<T> {
    /// Makes sure that the lock on the camera is being held for the lifetime of the stream,
    /// despite that the dmabuf streaming mechanism doesn't really need anything but the fd.
    /// Without this, it would be possible to acquire the same camera again even as the stream exists, as soon as the original camera goes out of scope.
    // TODO: make it clear that this impl always takes a system-wide exclusive lock over the device.
    camera: Arc<Mutex<dyn CameraImpl>>,
    stream: T,
}

impl StreamManual {
    pub fn finish(self, buf: dmabuf::DmaBufProtected)
        -> Result<(), (io::Error, dmabuf::DmaBufProtected, Self)>
    {
        let camera = self.camera;
        self.stream.finish(buf)
            .map_err(|(e, buf, stream)| (e, buf, Self { camera, stream}))
    }
}

impl<T> fmt::Debug for Stream<T> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "Stream for {:?}", Arc::as_ptr(&self.camera))
    }
}

impl<T> Deref for Stream<T> {
    type Target = T;
    fn deref(&self) -> &Self::Target {
        &self.stream
    }
}
impl<T> DerefMut for Stream<T> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.stream
    }
}

/** A representation of a camera that is ready to stream or streaming.

It may be modified even as the stream is running:
- controls may be set
- dynamic links may be changed
*/
pub struct AcquiredCamera(
    CameraId,
    // The actual Camera is in an Arc because it's shared with the Stream
    // The underlying camera device is always locked for writing.
    // AcquiredCamera cannot have multiple instances, so there's no point in wasting lock/unlock cycles and polluting code with short-lived guards in every method.
    ArcMutexGuard<RawMutex, dyn CameraImpl>,
);

impl AcquiredCamera {
    /// Returns globally unique, camera ID, stable for this device and for this software version (ideally for all software versions).
    pub fn get_id(&self) -> &CameraId {
        &self.0
    }
    
    // TODO: this should not be public, remove
    pub fn _video_capture_device(&self) -> &Device {
        self.1.video_capture_device()
    }
    
    /// Returns the camera controls.
    pub fn query_controls(&self) -> Result<Vec<control::Description>, io::Error> {
        let camera = &self.1;
        camera.video_capture_device().query_controls()
    }

    /// Reads the current value of the control.
    pub fn control(&self, control: &control::Description) -> io::Result<Control> {
        let camera = &self.1;
        camera.video_capture_device().control(control)
    }

    /// Sets the value of the control. May take effect after some frames.
    pub fn set_control(&self, control: control::Control) -> io::Result<()> {
        let camera = &self.1;
        camera.video_capture_device().set_control(control)
    }

    /// Verifies that the DeviceConfig belongs to this camera.
    fn unwrap_config(&self, config: DeviceConfig) -> Option<PipelineState> {
        if *config.device() == CameraHash::from(self.get_id()) {
            Some(config.config().clone())
        } else {
            None
        }
    }

    fn configure(&mut self, config: &PipelineState) -> TracedResult<(), subdev::Error> {
        // TODO: unconfigure conflicts.
        // 1. Refresh topology
        // 2. Create database
        // 3. Query for an unconflicting state given desired links and devices used by other apps
        // For a topology with a straight path, nothing is needed
        let camera = self.1.borrow_mut();
        let topology = camera.media_device()
            .get_topology()
            .map_err(subdev::Error::from)?;
        // TODO
        let topology_db = TopologyDatabase::<search::TextualUniverse>::new(&topology);
        
        let pad_pairs = config.pads.iter() // source, sink, source, ...
            .zip(
                config.pads[1..].iter()  // sink, source, sink, ...
            ) // (source, sink), (sink, source), (source, sink), ...
            .step_by(2); // (source, sink), (source, sink), ...
        
        let caps = camera.video_capture_device()
            .query_caps()
            .map_err(subdev::Error::from)?
            .capabilities;
            
        for (source, sink) in pad_pairs {
            // Not sure if the pad should be configured before, or after enabling the link
            let set_pad = |pad: &PadState| -> Result<(), subdev::Error> {
                let interface = search::entity_interface(&topology, source.id)
                    .and_then(|interface|
                        topology.0.interfaces.iter().find(|i| i.id == interface)
                    );
                if let Some(interface) = interface {
                    let path = crate::util::media::Io::interface_find_path(&mut media::Io, &interface)?;
                    let sio = &mut subdev::Io;
                    subdev::Io::open(sio, &path)?
                        .set_format(
                            sio,
                            // Presuming that this is the index relative to entity. Kernel docs don't make it clear, but it's kinda implied by calling this ioctl on the subdevice fd.
                            // https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/vidioc-subdev-enum-frame-size.html#c.V4L.v4l2_subdev_frame_size_enum
                            pad.pad_idx as u32,
                            MbusFrameFormat {
                                width: pad.width,
                                height: pad.height,
                                code: pad.mbus,
                                flags: MbusFrameFormatFlags::empty(),
                                field: 0,
                                colorspace: 0,
                                quantization: 0,
                                xfer_func: 0,
                            }
                        )?;
                }
                Ok(())
            };
            
            set_pad(source)?;
            set_pad(sink)?;
            
            trace!("Linking  from {:?} {:?} to {:?} {:?}", source.id, source.pad_idx, sink.id, sink.pad_idx);
            // Immutable links should not be a problem. The solver should avoid them in the first place (TODO), and immutable disable links are equivalent to no link (I think).
            // Immutable links only matter for removing conflicting links. That is TODO before this setup stage.
            match search::link_state(&topology_db, &source, &sink) {
                Some(LinkEnabled::Enabled) => trace!("Link already enabled"),
                Some(LinkEnabled::Disabled) => {
                    camera.media_device_mut().setup_link(MediaLinkDesc {
                        source: MediaPadDesc {
                            entity: source.id,
                            index: source.pad_idx,
                        },
                        sink: MediaPadDesc {
                            entity: sink.id,
                            index: sink.pad_idx,
                        },
                        state: LinkEnabled::Enabled,
                    }).map_trace(io::Error::from).map_trace_into()?;
                },
                None => {
                    Err(io::Error::other("Missing link").into()).with_trace()?
                },
            }
        }
        
        let camera = &self.1;
        let format = camera.video_capture_device()
            .format()
            .map_err(subdev::Error::from)?;
        let format = apply_video_config(&config.as_config(), format);
        camera.video_capture_device()
            .set_format(&format)
            .map_err(subdev::Error::from)?;
        // TODO: set capture interval
        Ok(())
    }
    
    fn configure_pipeline(&mut self, config: DeviceConfig) -> TracedResult<(), subdev::Error> {
        let config = self.unwrap_config(config)
            .ok_or(io::Error::other("Config for wrong camera"))
            .map_err(subdev::Error::from)?;
        self.configure(&config)
    }
    
    /// Starts recording from the camera
    pub fn start<'b>(&'b mut self, config: DeviceConfig, buffer_count: usize)
        -> TracedResult<Stream<dmabuf::Stream>, subdev::Error>
    {
        self.configure_pipeline(config)?;
        let camera = &self.1;
        Ok(Stream {
            camera: ArcMutexGuard::mutex(&camera).clone(),
            stream: camera.start(buffer_count).map_err(subdev::Error::from)?,
        })
    }
    
    /// Starts recording from the camera with manual buffer management
    pub fn start_manual(&mut self, config: DeviceConfig, buffer_count: usize)
        -> TracedResult<Stream<dmabuf::StreamManual>, subdev::Error>
    {
        self.configure_pipeline(config)?;
        let camera = &self.1;
        Ok(Stream {
            camera: ArcMutexGuard::mutex(&camera).clone(),
            stream: camera.stream_manual(buffer_count).map_err(subdev::Error::from)?,
        })
    }

    /// Returns a database of supported configs.
    pub fn create_configs_database<T: Database>(&mut self) -> io::Result<T> {
        //  TODO: The API of this method is fine, but the architecture is actually really awful.
        // The I/O parts should be separated from the ones that convert them into facts and ingest into a database. That would allow inspecting facts without adding special debug print modes here.
        /*
        General plan:
        1. Scan topology
        2. Save topology in database
        3. Find path through
        4. Acquire device
        5. Add pads
        6. Add mbus→fourcc on output
        7. Add output device
        8. Load config
        The output device may have resolution ranges. If the topology is no-subdevice, then variables are unconstrained and we're fucked without clp/fd.
        But UVC seems to not return ranges.
        Device-ful sensors also don't seem to return ranges, so constraining with inequalities should be good enough */

// 4. already done
        let device = self.1.media_device();
        // FIXME: topology should follow the path traversed to acquire the device. Store that path instead of discarding it.
        let topology = device.get_topology()?; // 1.
        let mut database = TopologyDatabase::new(&topology); // 2.

        database.add_facts("% ===== Subdevice information").unwrap();
        let mut subdev_io = subdev::Io;
        database.add_subdev_info(&mut subdev_io); // 5.

        // 6.
        // WARNING: this is complicated and maybe a bad choice.
        // There are 2 kinds of video devices in V4L: video device centric and media controller centric.
        // The advertised difference is that the first are controlled exclusively by setting propertis on the video device node, whereas the latter also on subdevices corresponding to the various processing entities.
        // Except there's no guarantee there that any entity is going to have a subdevice. That means it can't be controlled. There's also nothing saying the kernel can't decide what the devices are doing.
        // So the video-device-centric world is effectively a subset of the wider one. At least in theory - in practice, most media-based devices can just get hrdcoded. In this context, it doesn't make a lot of sense to split the code paths.
        // Except for the Mbus to FourCC conversions.
        // The video centered devices don't expose it at all.
        // But we need *something* there because we're creating an internal view of the actual pipeline and trying to fill the gaps with our best guesses. Without this conversion, we have no reasonable guess about the mbus code (okay, we do, but I'm too lazy to copy all the possible guesses).
        // We could create an internal view with "either concrete value or anything you like" as the mbus value, but I have a feeling it would just invite mistakes. So let's put in a gues.
        // And the dumbest guess and use "any". I mean, "FIXED". It doesn't matter, anyway. But it's enough to lock down the free variable to something concrete.
        if !self.1.video_capture_device()
            .query_caps()?
            .capabilities.intersects(capability::Flags::IO_MC)
        {
            let videodev = self.1.video_capture_entity();
            database.add_facts(&search::mbus_all_guesses(&videodev.name)).unwrap();
        }
        
        // 7.
        database.add_facts("% ===== Videodev frame sizes").unwrap();

        let confs = self.scan_configs()?;
        let videodev = self.1.video_capture_entity();
        for fact in search::framesizes_as_facts(&videodev.name, confs) {
            database.add_facts(&fact).unwrap();
        }

        // 8.
        database.add_facts("% ===== Stored device descriptions").unwrap();
        let mut config_io = FileConfig;
        for (source, facts) in config_io.device_definitions() {
            let res = facts
                .map_err(|e| tracing::error!("Device description file couldn't be loaded: {:?}.", e))
                .and_then(|facts|
                    database
                        .add_facts(&facts)
                        .map_err(|e| tracing::error!("Device description file contents couldn't be loaded: {:?}.", e))
                );
            match res {
                Err(()) => tracing::error!("The library will not work correctly. Check the contents of {}", source),
                Ok(()) => tracing::debug!("Loaded config from {}", source),
            };
        }

        Ok(database.into_database())
    }
    
    pub fn get_supported_configs(&mut self) -> io::Result<ConfigsDatabase> {
        let database = self.create_configs_database::<search::TextualUniverse>()?;
        Ok(ConfigsDatabase::new(
            self.get_id().into(),
            self.1.sensor_entity().name.clone(),
            database.into(),
        ))
    }
    
    /// Returns the advertised configs, consisting of format and sizes
    fn scan_configs(&mut self) -> io::Result<Vec<FrameSize>> {
        /* Returning more details for every possible or allowed configuration is not feasible here because the kernel interface doesn't present a lot of information to the user.
         * Testing each resolution doesn't make sense time-wise when the kernel returns a huge range of sizes like 1-65535.
         * Enumerating more information, e.g. supported frame intervals given a size, for the same reason. The size of the array to store all sizes with corresponding intervals becomes nonsensically big.
         * 
         * The kernel doesn't have to store all this information and can generate it out of an internal rule, but that rule isn't exposed to the user, so we would have to reconstruct it at the cost of a lot of computation: not practical.
         * 
         * Meanwhile, `fourcc`s are limited in number, and each query for sizes has to return a reasonably-sized data structure, so we can combine at least those two.
         * 
         * For now, we have to accept that some attempts to configure the device will fail and need to be retried if the device is misbehaving.
         * 
         * Maybe TODO: store the full manually-extracted rule the kernel is using in a per-device configuration file and load on-demand.
        */
        let mut framesizes = Vec::new();
        let d = self.1.video_capture_device();

        let formats = match d.enum_formats() {
            (list, Ok(_)) => list,
            (list, Err(e)) => if list.is_empty() {
                return Err(e);
            } else {
                list
            }
        };
        
        let mut last_error: Option<io::Error> = None;
        for f in formats {
            let (mut dims, res) = d.enum_framesizes(f.fourcc);
            if let Err(e) = res {
                warn!("Problem enumerating {:?}", f.fourcc);
                last_error = Some(e);
            }
            framesizes.append(&mut dims);
        }

        match (last_error, framesizes.is_empty()) {
            (Some(e), true) => Err(e),
            _ => Ok(framesizes),
        }
    }
}

/// A detected camera device.
///
/// *Note*: to avoid some complexity, querying controls requires acquiring the camera. If you need controls on a camera you can't acquire, please get in touch.
pub struct UnacquiredCamera {
    id: CameraId,
    device: Box<dyn UnacquiredCameraImpl>,
}

impl UnacquiredCamera {
    /// Returns globally unique, camera ID, stable for this device and for this software version (ideally for all software versions).
    pub fn get_id(&self) -> &CameraId {
        &self.id
    }

    // This fn is not &mut because we want to allow the user to query ID regardless of camera state.
    pub fn acquire(self) -> Result<AcquiredCamera, Error> {
        let device = self.device.acquire()?;
        Ok(AcquiredCamera(
            self.id,
            device.lock_arc(),
        ))
    }
}

/// A camera which has been detected, but not yet exclusively acquired for changing.
pub trait UnacquiredCameraImpl {
    /// Locks the camera for exlusive use, including the modification of its state.
    fn acquire(self: Box<Self>)
        -> Result<Arc<Mutex<dyn CameraImpl>>,  Error>;
}

/// An exclusive handle to a mutable camera resource.
///
/// Corresponds roughly to a sensor-device pipeline.
///
/// This is the main trait to fill in by camera vendors, defining all the low-level operations on the camera apart from acquiring.
///
/// Types implementing this impl are required to hold a system-wide exclusive lock over the hardware resource for as long as the instance exists. See `::util::flock::Locked`.
/// TODO: stop streaming when dropped? This is internal API, so maybe the owner type already takes care of stopping.
pub trait CameraImpl: Send + Sync {
    /// Start streaming
    fn start(&self, buffer_count: usize) -> Result<dmabuf::Stream, io::Error> {
        dmabuf::Stream::with_buffers(
            self.video_capture_device(),
            buffer::Type::VideoCapture,
            buffer_count as u32,
        )
    }
    /// Create a stream with manual buffer control.
    fn stream_manual(&self, buffer_count: usize) -> Result<dmabuf::StreamManual, io::Error> {
        dmabuf::StreamManual::new(
            self.video_capture_device(),
            buffer::Type::VideoCapture,
            buffer_count as u32,
        )
    }

    /// Return the video capture device. Private interface
    fn video_capture_device(&self) -> &v4l::Device;
    
    /// Return the media device
    fn media_device(&self) -> &media::Device;
    
    /// Return the media device
    fn media_device_mut(&mut self) -> &mut media::Device;
    
    // TODO: probably should be removed in favor of the active path
    fn video_capture_entity(&self) -> &MediaV2Entity;
    // TODO: probably should be removed in favor of the active path
    fn sensor_entity(&self) -> &MediaV2Entity;
}

/// This trait is only responsible for releasing the lock when dropped
pub trait Lock {}

pub type Builder = fn(camera: CameraInfo)
    -> Result<UnacquiredCamera, Error>;

pub type CheckFn = for<'a> fn(&'a udev::Device)
    -> Vec<CreationKit>;

/// Put the entry function to every supported kind of camera here.
pub const PIPELINES: &'static [CheckFn] = &[
    uvc::check_match,
];