Image.h

Go to the documentation of this file.

//
// Notice
// ALL NVIDIA DESIGN SPECIFICATIONS AND CODE ("MATERIALS") ARE PROVIDED "AS IS" NVIDIA MAKES
// NO REPRESENTATIONS, WARRANTIES, EXPRESSED, IMPLIED, STATUTORY, OR OTHERWISE WITH RESPECT TO
// THE MATERIALS, AND EXPRESSLY DISCLAIMS ANY IMPLIED WARRANTIES OF NONINFRINGEMENT,
// MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
//
// NVIDIA CORPORATION & AFFILIATES assumes no responsibility for the consequences of use of such
// information or for any infringement of patents or other rights of third parties that may
// result from its use. No license is granted by implication or otherwise under any patent
// or patent rights of NVIDIA CORPORATION & AFFILIATES. No third party distribution is allowed unless
// expressly authorized by NVIDIA. Details are subject to change without notice.
// This code supersedes and replaces all information previously supplied.
// NVIDIA CORPORATION & AFFILIATES products are not authorized for use as critical
// components in life support devices or systems without express written approval of
// NVIDIA CORPORATION & AFFILIATES.
//
// SPDX-FileCopyrightText: Copyright (c) 2016-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: LicenseRef-NvidiaProprietary
//
// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
// property and proprietary rights in and to this material, related
// documentation and any modifications thereto. Any use, reproduction,
// disclosure or distribution of this material and related documentation
// without an express license agreement from NVIDIA CORPORATION or
// its affiliates is strictly prohibited.
//

#ifndef DW_IMAGE_IMAGE_H_
#define DW_IMAGE_IMAGE_H_

#include <dw/core/Config.h>
#include <dw/core/base/Exports.h>
#include <dw/core/base/Types.h>
#include <dw/core/base/Status.h>
#include <dw/core/context/Context.h>

// Forward defines from GL
// Note: GL defines the specific bitdepth of these types
//       so it is safe to declare then here without including GL.h
typedef uint32_t GLenum;
typedef uint32_t GLuint;

#if (defined(__cplusplus) && (defined(LINUX) || defined(VIBRANTE)))
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
#endif

#include <cuda_runtime.h>

#if (defined(__cplusplus) && (defined(LINUX) || defined(VIBRANTE)))
#pragma GCC diagnostic pop
#endif

#ifdef VIBRANTE
#include <sys/time.h>
#include <nvmedia_image.h>
#include <nvmedia_isp_stat.h>
#endif

#ifdef __cplusplus
extern "C" {
#endif

#define DW_MAX_IMAGE_PLANES 3
#define DW_ISP_MAX_COLOR_COMPONENT (4U)

#define DW_MAX_NUM_TEMPERATURES 4

typedef enum dwImageType {
    DW_IMAGE_CPU  = 0,
    DW_IMAGE_CUDA = 2,
#ifdef VIBRANTE
    DW_IMAGE_NVMEDIA = 3,
#endif

    DW_IMAGE_GL = 0xFF
} dwImageType;

typedef struct dwImageObject* dwImageHandle_t;
typedef struct dwImageObject const* dwConstImageHandle_t;
typedef struct dwImageAllocationAttrList* dwImageAllocationAttrListHandle_t;

typedef struct dwImagePool
{
    dwImageHandle_t* images;
    size_t imageCount;
} dwImagePool;

typedef enum dwImageFormat {
    DW_IMAGE_FORMAT_UNKNOWN = 0,

    // Interleaved formats
    DW_IMAGE_FORMAT_R_INT16 = 900,

    DW_IMAGE_FORMAT_R_UINT8 = 1000,
    DW_IMAGE_FORMAT_R_UINT16,
    DW_IMAGE_FORMAT_R_UINT32,
    DW_IMAGE_FORMAT_R_FLOAT16,
    DW_IMAGE_FORMAT_R_FLOAT32,

    DW_IMAGE_FORMAT_RG_INT16,
    DW_IMAGE_FORMAT_RG_UINT8,
    DW_IMAGE_FORMAT_RG_FLOAT32,

    DW_IMAGE_FORMAT_RGB_UINT8,
    DW_IMAGE_FORMAT_RGB_UINT16,
    DW_IMAGE_FORMAT_RGB_FLOAT16,
    DW_IMAGE_FORMAT_RGB_FLOAT32,

    DW_IMAGE_FORMAT_RGBA_UINT8,
    DW_IMAGE_FORMAT_RGBA_UINT16,
    DW_IMAGE_FORMAT_RGBA_FLOAT16,
    DW_IMAGE_FORMAT_RGBA_FLOAT32,

    DW_IMAGE_FORMAT_RGBX_FLOAT16 = 1200,

    // interleaved YUV444 format
    DW_IMAGE_FORMAT_VUYX_UINT8,
    DW_IMAGE_FORMAT_VUYX_UINT16,

    // Planar formats
    DW_IMAGE_FORMAT_RGB_UINT8_PLANAR = 2000,
    DW_IMAGE_FORMAT_RGB_UINT16_PLANAR,
    DW_IMAGE_FORMAT_RGB_FLOAT16_PLANAR,
    DW_IMAGE_FORMAT_RGB_FLOAT32_PLANAR,

    DW_IMAGE_FORMAT_RCB_FLOAT16_PLANAR,
    DW_IMAGE_FORMAT_RCB_FLOAT32_PLANAR,

    DW_IMAGE_FORMAT_RCC_FLOAT16_PLANAR,
    DW_IMAGE_FORMAT_RCC_FLOAT32_PLANAR,

    DW_IMAGE_FORMAT_YUV420_UINT8_PLANAR = 3000,
    DW_IMAGE_FORMAT_YUV420_UINT8_SEMIPLANAR,
    DW_IMAGE_FORMAT_YUV420_UINT16_SEMIPLANAR,
    DW_IMAGE_FORMAT_YUV422_UINT8_SEMIPLANAR = 3100,

    // planar YUV 444
    DW_IMAGE_FORMAT_YUV_UINT8_PLANAR,
    DW_IMAGE_FORMAT_YUV_UINT16_PLANAR,

    DW_IMAGE_FORMAT_RAW_UINT16 = 4000,
    DW_IMAGE_FORMAT_RAW_FLOAT16,

} dwImageFormat;

typedef enum {
    DW_IMAGE_MEMORY_TYPE_DEFAULT = 0,
    DW_IMAGE_MEMORY_TYPE_PITCH = 1,
    DW_IMAGE_MEMORY_TYPE_BLOCK = 2,

} dwImageMemoryType;

typedef struct dwImageDataLines
{
    uint32_t bytesPerLine;

    dwVector2ui embeddedDataSize;

    uint8_t* topLineData;
    uint8_t* bottomLineData;
} dwImageDataLines;

typedef enum {

    DW_IMAGE_FLAGS_EMBEDDED_LINES = (1 << 2),

    DW_IMAGE_FLAGS_SENSOR_SETTINGS = (1 << 3),

    DW_IMAGE_FLAGS_FRAME_SEQUENCE_NUMBER = (1 << 4),

    DW_IMAGE_FLAGS_SYSMEM = (1 << 5),

    DW_IMAGE_FLAGS_HAS_RAW_ORDER_DESCRIPTOR = (1 << 6),

    DW_IMAGE_FLAGS_CONTROLINFO = (1 << 7),

    DW_IMAGE_FLAGS_LUMINANCE_CALIBRATED = (1 << 8),

    DW_IMAGE_FLAGS_TOTAL_WHITE_BALANCE_GAIN = (1 << 9),

    DW_IMAGE_FLAGS_GTM_SPLINE_INFO = (1 << 10),

    DW_IMAGE_FLAGS_SENSOR_TEMPERATURE = (1 << 11),
} dwImageMetaDataFlags;

typedef struct dwImageSplineControlPoint
{
    float32_t x;
    float32_t y;
    float64_t slope;
} dwImageSplineControlPoint;

typedef struct dwImageSensorStatistics
{
    float32_t exposureTime;

    float32_t analogGain;

    float32_t conversionGain;

    float32_t digitalGain;

    float32_t wbGain[4];

    float32_t alpha;

    float64_t luminanceCalibrationFactor;

    float32_t autoAwbGainGain[DW_ISP_MAX_COLOR_COMPONENT];

    float32_t cct;

    float32_t brightnessKey;

    float32_t rawImageMidTone;

    uint8_t numTemperatures;
    float32_t sensorTempCelsius[DW_MAX_NUM_TEMPERATURES];
} dwImageSensorStatistics;

typedef struct dwImageRawProperties
{
    uint32_t msbPosition;

    uint32_t rawFormatDescriptor;

    uint32_t rawBitDataType;
} dwImageRawProperties;

typedef struct dwImageMetaData
{
    uint32_t flags;

    float32_t exposureTime
        DW_DEPRECATED("This structure variable is deprecated and will be removed in the next major release. Please use the new variable in sensorStatistics in this struct");

    float32_t analogGain
        DW_DEPRECATED("This structure variable is deprecated and will be removed in the next major release. Please use the new variable in sensorStatistics in this struct");

    float32_t conversionGain
        DW_DEPRECATED("This structure variable is deprecated and will be removed in the next major release. Please use the new variable in sensorStatistics in this struct");

    float32_t digitalGain
        DW_DEPRECATED("This structure variable is deprecated and will be removed in the next major release. Please use the new variable in sensorStatistics in this struct");

    float32_t wbGain[4] DW_DEPRECATED("This structure variable is deprecated and will be removed in the next major release. Please use the new variable in sensorStatistics in this struct");

    uint32_t msbPosition
        DW_DEPRECATED("This structure variable is deprecated and will be removed in the next major release. Please use the new variable in rawProperties in this struct");

    uint32_t frameSequenceNumber;

    dwImageDataLines dataLines;

    // Holds properties unique to raw images
    dwImageRawProperties rawProperties;

    // All paramters from here are read-only

    dwImageSensorStatistics sensorStatistics;

    dwImageSplineControlPoint gtmSplineControlPoint[18U];

    dwImageAllocationAttrListHandle_t allocAttrs;
} dwImageMetaData;

typedef struct dwImageProperties
{
    dwImageType type;
    uint32_t width;
    uint32_t height;
    dwImageFormat format;
    dwImageMetaData meta;
    dwImageMemoryType memoryLayout;
} dwImageProperties;

typedef struct dwImageCPU
{
    dwImageProperties prop;
    size_t pitch[DW_MAX_IMAGE_PLANES];
    uint8_t* data[DW_MAX_IMAGE_PLANES];
    dwTime_t timestamp_us;
} dwImageCPU;

typedef struct dwImageCUDA
{
    dwImageProperties prop;
    size_t pitch[DW_MAX_IMAGE_PLANES]; // pitch in bytes
    void* dptr[DW_MAX_IMAGE_PLANES];
    cudaArray_t array[DW_MAX_IMAGE_PLANES];
    dwTime_t timestamp_us;
} dwImageCUDA;

#ifdef VIBRANTE
typedef struct dwImageNvMedia
{
    dwImageProperties prop;
    NvMediaImage* img
        DW_DEPRECATED("NvMediaImage is deprecated and will be removed in the next major release.");
    dwTime_t timestamp_us;
} dwImageNvMedia;
#endif

DW_API_PUBLIC
dwStatus dwImage_create(dwImageHandle_t* const image,
                        dwImageProperties properties,
                        dwContextHandle_t const ctx);

DW_API_PUBLIC
dwStatus dwImage_createAndBindBuffer(dwImageHandle_t* const image,
                                     dwImageProperties properties,
                                     void* const buffersIn[DW_MAX_IMAGE_PLANES],
                                     size_t const pitches[DW_MAX_IMAGE_PLANES], size_t const bufferCount,
                                     dwContextHandle_t const ctx);

DW_API_PUBLIC
dwStatus dwImage_createAndBindCUDAArray(dwImageHandle_t* const image,
                                        dwImageProperties properties,
                                        cudaArray_t const buffers[DW_MAX_IMAGE_PLANES], size_t const bufferCount,
                                        dwContextHandle_t const ctx);

#ifdef VIBRANTE

DW_API_PUBLIC
dwStatus dwImage_createAndBindNvMedia(dwImageHandle_t* image, NvMediaImage* nvmImage);
#endif

DW_API_PUBLIC
dwStatus dwImage_destroy(dwImageHandle_t const image);

// specifiers

DW_API_PUBLIC
dwStatus dwImage_getProperties(dwImageProperties* const properties, dwConstImageHandle_t const image);

DW_API_PUBLIC
dwStatus dwImage_getTimestamp(dwTime_t* const timestamp, dwConstImageHandle_t const image);

DW_API_PUBLIC
dwStatus dwImage_setTimestamp(dwTime_t const timestamp, dwImageHandle_t const image);

DW_API_PUBLIC
dwStatus dwImage_getMetaData(dwImageMetaData* const metaData, dwConstImageHandle_t const image);

DW_API_PUBLIC
dwStatus dwImage_setMetaData(dwImageMetaData const* const metaData, dwImageHandle_t const image);

DW_API_PUBLIC
dwStatus dwImage_getCPU(dwImageCPU** const imageCPU, dwImageHandle_t const image);

DW_API_PUBLIC
dwStatus dwImage_getCUDA(dwImageCUDA** const imageCUDA, dwImageHandle_t const image);

#ifdef VIBRANTE

DW_API_PUBLIC
dwStatus dwImage_getNvMedia(dwImageNvMedia** imageNvMedia, dwImageHandle_t image);
#endif

DW_API_PUBLIC
dwStatus dwImage_getPixelType(dwTrivialDataType* const type, dwImageFormat const format);

DW_API_PUBLIC
dwStatus dwImage_getPlaneCount(size_t* const planeCount, dwImageFormat const format);

DW_API_PUBLIC
dwStatus dwImage_copyConvert(dwImageHandle_t const output, dwConstImageHandle_t const input, dwContextHandle_t const context);

DW_API_PUBLIC
dwStatus dwImage_copyConvertAsync(dwImageHandle_t const output, dwConstImageHandle_t const input, cudaStream_t const stream, dwContextHandle_t const context);

DW_API_PUBLIC dwStatus dwImageCUDA_getPlaneAsImage(dwImageCUDA* const planeImage,
                                                   dwImageCUDA const* const srcImage,
                                                   uint32_t const planeIdx);

DW_API_PUBLIC dwStatus dwImageCUDA_mapToROI(dwImageCUDA* const dstImg, dwImageCUDA const* const srcImg, dwRect const roi);

DW_API_PUBLIC dwStatus dwImage_getDataLayout(size_t* const elementSize,
                                             size_t* const planeCount,
                                             uint32_t planeChannelCount[DW_MAX_IMAGE_PLANES],
                                             dwVector2ui planeSize[DW_MAX_IMAGE_PLANES],
                                             dwImageProperties const* const prop);

#ifdef __cplusplus
}
#endif

#endif // DW_IMAGE_IMAGE_H_