api/inference.h ¶

Namespaces ¶

Name
cvedia
cvedia::rt API for controlling the server and lifecycle management of runtime instances.
cvedia::rt::iface
cvedia::rt::api
cvedia::rt::api::inference

Classes ¶

	Name
struct	cvedia::rt::api::inference::PollResult

Functions ¶

	Name
void	registerPreprocessHandler(std::string const & name, module::preprocessHandler func)
void	registerPreprocessBufferHandler(std::string const & name, module::preprocessBufferHandler func)
void	registerPostprocessHandler(std::string const & name, module::postprocessHandler func)
void	registerPostprocessHandler(std::string const & name, module::postprocessCudaHandler func)
void	registerPostprocessHandler(std::string const & name, module::postprocessOpenClHandler func)
void	registerSchemeHandler(std::string const & protocol, module::createInferenceHandler func)
void	registerExtHandler(std::string const & extension, module::createInferenceHandler func)
void	registerInferenceBackendHandler(std::string const & uriPattern, module::createInferenceEngine func)
NODISCARD expected< module::preprocessHandler >	getPreProcessHandler(std::string const & handler)
NODISCARD expected< module::preprocessBufferHandler >	getPreProcessBufferHandler(std::string const & handler)
NODISCARD expected< module::postprocessHandler >	getPostProcessHandler(std::string const & handler)
NODISCARD expected< module::postprocessCudaHandler >	getPostProcessCudaHandler(std::string const & handler)
NODISCARD bool	hasPostProcessCudaHandler(std::string const & handler)
NODISCARD expected< module::postprocessOpenClHandler >	getPostProcessOpenClHandler(std::string const & handler)
NODISCARD bool	hasPostProcessOpenClHandler(std::string const & handler)
NODISCARD expected< internal::InferenceRegistry::ResolvedPostprocess >	getPostProcessHandler(std::string const & handler, internal::InferenceRegistry::PostPreference pref)
NODISCARD expected< module::createInferenceHandler >	getSchemeHandler(std::string const & backend)
NODISCARD expected< module::createInferenceHandler >	getExtHandler(std::string const & backend)
NODISCARD expected< module::createInferenceEngine >	getInferenceBackendHandler(std::string const & modelUri)
NODISCARD expected< cvec >	runInference(std::string const & modelUri, std::string const & solutionId, cvec const & jobs)
NODISCARD expected< cvec >	runInference(std::string const & modelUri, std::string const & solutionId, cbuffer const & buffer)
NODISCARD expected< cvec >	runInference(std::string const & modelUri, std::string const & solutionId, std::vector< cbuffer > const & buffers)
NODISCARD expected< std::pair< size_t, size_t > >	queryModel(std::string const & modelUri)
NODISCARD expected< ssize_t >	queryModelBatchSize(std::string const & modelUri)
NODISCARD expected< pCValue >	getModelConfig(std::string const & modelUri)
NODISCARD expected< std::vector< unsigned long long > >	runInferenceAsync(std::string const & modelUri, std::string const & solutionId, cvec const & jobs)
NODISCARD expected< std::vector< unsigned long long > >	runInferenceAsync(std::string const & modelUri, std::string const & solutionId, cbuffer const & buffer)
NODISCARD expected< std::vector< unsigned long long > >	runInferenceAsync(std::string const & modelUri, std::string const & solutionId, std::vector< cbuffer > const & buffers)
NODISCARD expected< InferenceFanoutJobIds >	runInferenceFanoutAsync(std::vector< InferenceFanoutConsumer > const & consumers)
NODISCARD expected< std::vector< unsigned long long > >	runInferenceAsync(std::string const & modelUri, std::string const & solutionId, std::vector< cbuffer > const & buffers, cvec const & jobs)
NODISCARD expected< cvec >	getInferenceResults(std::string const & modelUri, std::vector< unsigned long long > const & jobIds)
NODISCARD expected< PollResult >	pollInferenceResults(std::vector< unsigned long long > const & jobIds)
NODISCARD expected< cvec >	runPackedInference(std::string const & modelUri, std::string const & solutionId, std::vector< cbuffer > const & buffers, cvec const & jobs, pCValue const & options =nullptr)
NODISCARD expected< cvec >	runPackedInference(std::string const & modelUri, std::string const & solutionId, cvec const & jobs, pCValue const & options =nullptr)
NODISCARD expected< std::shared_ptr< iface::InferenceEngine > >	loadModel(std::string const & uri, std::string const & name)
NODISCARD expected< std::shared_ptr< iface::InferenceEngine > >	loadModel(std::string const & uri, std::string const & name, int poolSize, pCValue const & handlerConfig)
NODISCARD std::vector< std::pair< std::string, std::string > >	getAllDevices()
NODISCARD std::vector< std::string >	getAllSchemeHandlers()
NODISCARD std::map< std::string, pCValue >	getAllCapabilities()
NODISCARD expected< std::pair< cbuffer, std::vector< Rect2f > > >	packJobs(cvec const & jobs, int width, int height, int packingInterpolationMethod =0)
void	removeAllHandlers()
void	invalidateDeviceCache()
unsigned	deviceCacheGeneration()
NODISCARD std::vector< std::string >	findModels(std::string const & uri)
NODISCARD expected< module::ModelPaths >	resolveUri(std::string const & uri)
NODISCARD bool	isDeviceDenied(std::string const & modelUri, std::string const & deviceId)
void	setLastModelLoadError(std::string const & modelUri, std::string const & message)
NODISCARD std::optional< std::string >	getLastModelLoadError(std::string const & modelUri)
void	clearLastModelLoadError(std::string const & modelUri)

Functions Documentation ¶

function registerPreprocessHandler ¶

void registerPreprocessHandler(
    std::string const & name,
    module::preprocessHandler func
)

function registerPreprocessBufferHandler ¶

void registerPreprocessBufferHandler(
    std::string const & name,
    module::preprocessBufferHandler func
)

function registerPostprocessHandler ¶

void registerPostprocessHandler(
    std::string const & name,
    module::postprocessHandler func
)

function registerPostprocessHandler ¶

void registerPostprocessHandler(
    std::string const & name,
    module::postprocessCudaHandler func
)

function registerPostprocessHandler ¶

void registerPostprocessHandler(
    std::string const & name,
    module::postprocessOpenClHandler func
)

function registerSchemeHandler ¶

void registerSchemeHandler(
    std::string const & protocol,
    module::createInferenceHandler func
)

function registerExtHandler ¶

void registerExtHandler(
    std::string const & extension,
    module::createInferenceHandler func
)

function registerInferenceBackendHandler ¶

void registerInferenceBackendHandler(
    std::string const & uriPattern,
    module::createInferenceEngine func
)

function getPreProcessHandler ¶

NODISCARD expected< module::preprocessHandler > getPreProcessHandler(
    std::string const & handler
)

function getPreProcessBufferHandler ¶

NODISCARD expected< module::preprocessBufferHandler > getPreProcessBufferHandler(
    std::string const & handler
)

function getPostProcessHandler ¶

NODISCARD expected< module::postprocessHandler > getPostProcessHandler(
    std::string const & handler
)

function getPostProcessCudaHandler ¶

NODISCARD expected< module::postprocessCudaHandler > getPostProcessCudaHandler(
    std::string const & handler
)

function hasPostProcessCudaHandler ¶

NODISCARD bool hasPostProcessCudaHandler(
    std::string const & handler
)

function getPostProcessOpenClHandler ¶

NODISCARD expected< module::postprocessOpenClHandler > getPostProcessOpenClHandler(
    std::string const & handler
)

function hasPostProcessOpenClHandler ¶

NODISCARD bool hasPostProcessOpenClHandler(
    std::string const & handler
)

function getPostProcessHandler ¶

NODISCARD expected< internal::InferenceRegistry::ResolvedPostprocess > getPostProcessHandler(
    std::string const & handler,
    internal::InferenceRegistry::PostPreference pref
)

function getSchemeHandler ¶

NODISCARD expected< module::createInferenceHandler > getSchemeHandler(
    std::string const & backend
)

function getExtHandler ¶

NODISCARD expected< module::createInferenceHandler > getExtHandler(
    std::string const & backend
)

function getInferenceBackendHandler ¶

NODISCARD expected< module::createInferenceEngine > getInferenceBackendHandler(
    std::string const & modelUri
)

function runInference ¶

NODISCARD expected< cvec > runInference(
    std::string const & modelUri,
    std::string const & solutionId,
    cvec const & jobs
)

function runInference ¶

NODISCARD expected< cvec > runInference(
    std::string const & modelUri,
    std::string const & solutionId,
    cbuffer const & buffer
)

function runInference ¶

NODISCARD expected< cvec > runInference(
    std::string const & modelUri,
    std::string const & solutionId,
    std::vector< cbuffer > const & buffers
)

function queryModel ¶

NODISCARD expected< std::pair< size_t, size_t > > queryModel(
    std::string const & modelUri
)

function queryModelBatchSize ¶

NODISCARD expected< ssize_t > queryModelBatchSize(
    std::string const & modelUri
)

function getModelConfig ¶

NODISCARD expected< pCValue > getModelConfig(
    std::string const & modelUri
)

function runInferenceAsync ¶

NODISCARD expected< std::vector< unsigned long long > > runInferenceAsync(
    std::string const & modelUri,
    std::string const & solutionId,
    cvec const & jobs
)

function runInferenceAsync ¶

NODISCARD expected< std::vector< unsigned long long > > runInferenceAsync(
    std::string const & modelUri,
    std::string const & solutionId,
    cbuffer const & buffer
)

function runInferenceAsync ¶

NODISCARD expected< std::vector< unsigned long long > > runInferenceAsync(
    std::string const & modelUri,
    std::string const & solutionId,
    std::vector< cbuffer > const & buffers
)

function runInferenceFanoutAsync ¶

NODISCARD expected< InferenceFanoutJobIds > runInferenceFanoutAsync(
    std::vector< InferenceFanoutConsumer > const & consumers
)

Queue several model consumers of shared source buffers. All source buffers are preflighted before the first worker submission, allowing transports with explicit ownership to reserve the complete fan-out. This makes ownership preflight atomic only; worker submission may be partial. On error, do not retry the same opaque frame: an earlier consumer may own it and unused references remain lease-owned until teardown.

function runInferenceAsync ¶

NODISCARD expected< std::vector< unsigned long long > > runInferenceAsync(
    std::string const & modelUri,
    std::string const & solutionId,
    std::vector< cbuffer > const & buffers,
    cvec const & jobs
)

Parameters:

modelUri Model URI for inference
solutionId Solution ID
buffers Source buffers (may be ephemeral views) - paired with jobs by index
jobs Job metadata (x, y, width, height, trackId, etc.) - NO source field

Return: Vector of job IDs for polling results

Run async inference with buffers passed separately from job metadata. Buffers bypass CValue storage to allow ephemeral decoder surfaces. Buffer copy happens AFTER QoS acceptance in the dispatcher.

function getInferenceResults ¶

NODISCARD expected< cvec > getInferenceResults(
    std::string const & modelUri,
    std::vector< unsigned long long > const & jobIds
)

function pollInferenceResults ¶

NODISCARD expected< PollResult > pollInferenceResults(
    std::vector< unsigned long long > const & jobIds
)

Parameters:

jobIds Job IDs to check for completion

Return: PollResult containing:

completed: Vector of (jobId, result) pairs for completed jobs
pendingJobIds: Vector of job IDs still pending (valid future, not yet ready)

Non-blocking poll for completed inference results. Returns immediately with any results that are ready, without waiting.

Jobs that failed, were not found, or had invalid futures are NOT included in pendingJobIds. Use this for responsive async inference patterns where you want to process results as they become available without blocking.

function runPackedInference ¶

NODISCARD expected< cvec > runPackedInference(
    std::string const & modelUri,
    std::string const & solutionId,
    std::vector< cbuffer > const & buffers,
    cvec const & jobs,
    pCValue const & options =nullptr
)

Parameters:

modelUri Model URI for inference
solutionId Solution ID
buffers Source buffers (may be ephemeral views)
jobs Array of job metadata (x, y, width, height, userData) - NO source field
options Optional configuration (e.g., edgeFilter settings)

Return: Array of detection results (one array per job), maintaining input order

Run inference with central texture packing across multiple streams. Buffers passed separately to avoid ephemeral buffer storage in CValue.

Buffers and jobs are paired by index. This function blocks until all jobs complete.

function runPackedInference ¶

NODISCARD expected< cvec > runPackedInference(
    std::string const & modelUri,
    std::string const & solutionId,
    cvec const & jobs,
    pCValue const & options =nullptr
)

Parameters:

modelUri Model URI for inference
solutionId Solution ID
jobs Array of job objects with metadata
options Optional configuration (e.g., edgeFilter settings)

Return: Array of detection results (one array per job), maintaining input order

Run inference with central texture packing across multiple streams. Jobs are grouped by transport URI and packed into shared atlases.

This function blocks until all jobs complete. The background dispatcher accumulates jobs from multiple concurrent callers and batches them by transport URI for efficient packing.

function loadModel ¶

NODISCARD expected< std::shared_ptr< iface::InferenceEngine > > loadModel(
    std::string const & uri,
    std::string const & name
)

function loadModel ¶

NODISCARD expected< std::shared_ptr< iface::InferenceEngine > > loadModel(
    std::string const & uri,
    std::string const & name,
    int poolSize,
    pCValue const & handlerConfig
)

function getAllDevices ¶

NODISCARD std::vector< std::pair< std::string, std::string > > getAllDevices()

function getAllSchemeHandlers ¶

NODISCARD std::vector< std::string > getAllSchemeHandlers()

function getAllCapabilities ¶

NODISCARD std::map< std::string, pCValue > getAllCapabilities()

function packJobs ¶

NODISCARD expected< std::pair< cbuffer, std::vector< Rect2f > > > packJobs(
    cvec const & jobs,
    int width,
    int height,
    int packingInterpolationMethod =0
)

function removeAllHandlers ¶

void removeAllHandlers()

function invalidateDeviceCache ¶

void invalidateDeviceCache()

function deviceCacheGeneration ¶

unsigned deviceCacheGeneration()

function findModels ¶

NODISCARD std::vector< std::string > findModels(
    std::string const & uri
)

function resolveUri ¶

NODISCARD expected< module::ModelPaths > resolveUri(
    std::string const & uri
)

function isDeviceDenied ¶

NODISCARD bool isDeviceDenied(
    std::string const & modelUri,
    std::string const & deviceId
)

function setLastModelLoadError ¶

void setLastModelLoadError(
    std::string const & modelUri,
    std::string const & message
)

function getLastModelLoadError ¶

NODISCARD std::optional< std::string > getLastModelLoadError(
    std::string const & modelUri
)

function clearLastModelLoadError ¶

void clearLastModelLoadError(
    std::string const & modelUri
)