rte_dmadev.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2021 HiSilicon Limited
 * Copyright(c) 2021 Intel Corporation
 * Copyright(c) 2021 Marvell International Ltd
 * Copyright(c) 2021 SmartShare Systems
 */

#ifndef RTE_DMADEV_H
#define RTE_DMADEV_H

#include <stdint.h>
#include <errno.h>

#include <rte_bitops.h>
#include <rte_common.h>
#include <rte_uuid.h>

#ifdef __cplusplus
extern "C" {
#endif

#define RTE_DMADEV_DEFAULT_MAX 64

int rte_dma_dev_max(size_t dev_max);

int rte_dma_get_dev_id_by_name(const char *name);

bool rte_dma_is_valid(int16_t dev_id);

uint16_t rte_dma_count_avail(void);

int16_t rte_dma_next_dev(int16_t start_dev_id);

#define RTE_DMA_FOREACH_DEV(p) \
    for (p = rte_dma_next_dev(0); \
         p != -1; \
         p = rte_dma_next_dev(p + 1))


#define RTE_DMA_CAPA_MEM_TO_MEM     RTE_BIT64(0)

#define RTE_DMA_CAPA_MEM_TO_DEV     RTE_BIT64(1)

#define RTE_DMA_CAPA_DEV_TO_MEM     RTE_BIT64(2)

#define RTE_DMA_CAPA_DEV_TO_DEV     RTE_BIT64(3)

#define RTE_DMA_CAPA_SVA                RTE_BIT64(4)

#define RTE_DMA_CAPA_SILENT             RTE_BIT64(5)

#define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6)

#define RTE_DMA_CAPA_M2D_AUTO_FREE      RTE_BIT64(7)

#define RTE_DMA_CAPA_PRI_POLICY_SP  RTE_BIT64(8)

#define RTE_DMA_CAPA_INTER_PROCESS_DOMAIN   RTE_BIT64(9)

#define RTE_DMA_CAPA_INTER_OS_DOMAIN        RTE_BIT64(10)

#define RTE_DMA_CAPA_OPS_COPY           RTE_BIT64(32)

#define RTE_DMA_CAPA_OPS_COPY_SG    RTE_BIT64(33)

#define RTE_DMA_CAPA_OPS_FILL       RTE_BIT64(34)

#define RTE_DMA_CAPA_OPS_ENQ_DEQ    RTE_BIT64(35)

#define RTE_DMA_CFG_FLAG_SILENT RTE_BIT64(0)

#define RTE_DMA_CFG_FLAG_ENQ_DEQ RTE_BIT64(1)

struct rte_dma_info {
    const char *dev_name; 
    uint64_t dev_capa;
    uint16_t max_vchans;
    uint16_t max_desc;
    uint16_t min_desc;
    uint16_t max_sges;
    int16_t numa_node;
    uint16_t nb_vchans;
    uint16_t nb_priorities;
};

int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info);

struct rte_dma_conf {
    uint16_t nb_vchans;
    /* The priority of the DMA device.
     * This value should be lower than the field 'nb_priorities' of struct
     * rte_dma_info which get from rte_dma_info_get(). If the DMA device
     * does not support priority scheduling, this value should be zero.
     *
     * Lowest value indicates higher priority and vice-versa.
     */
    uint16_t priority;
    uint64_t flags;
};

int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf);

int rte_dma_start(int16_t dev_id);

int rte_dma_stop(int16_t dev_id);

int rte_dma_close(int16_t dev_id);

enum rte_dma_direction {
    RTE_DMA_DIR_MEM_TO_MEM,
    RTE_DMA_DIR_MEM_TO_DEV,
    RTE_DMA_DIR_DEV_TO_MEM,
    RTE_DMA_DIR_DEV_TO_DEV,
};

enum rte_dma_port_type {
    RTE_DMA_PORT_NONE,
    RTE_DMA_PORT_PCIE, 
};

struct rte_dma_port_param {
    enum rte_dma_port_type port_type;
    union {
        __extension__
        union {
            struct {
                uint64_t coreid : 4; 
                uint64_t pfid : 8; 
                uint64_t vfen : 1; 
                uint64_t vfid : 16; 
                uint64_t pasid : 20;
                uint64_t attr : 3;
                uint64_t ph : 2;
                uint64_t st : 16;
            };
            uint64_t val;
        } pcie;
    };
    uint64_t reserved[2]; 
};

struct rte_dma_auto_free_param {
    union {
        struct {
            struct rte_mempool *pool;
        } m2d;
    };
    uint64_t reserved[2];
};

enum rte_dma_inter_domain_type {
    RTE_DMA_INTER_DOMAIN_NONE,
    RTE_DMA_INTER_PROCESS_DOMAIN,
    RTE_DMA_INTER_OS_DOMAIN,
};

struct rte_dma_inter_domain_param {
    enum rte_dma_inter_domain_type type;
    uint16_t src_handler;
    uint16_t dst_handler;
    uint64_t reserved[2];
};

struct rte_dma_vchan_conf {
    enum rte_dma_direction direction;
    uint16_t nb_desc;
    struct rte_dma_port_param src_port;
    struct rte_dma_port_param dst_port;
    struct rte_dma_auto_free_param auto_free;
    struct rte_dma_inter_domain_param domain;
};

int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan,
            const struct rte_dma_vchan_conf *conf);

struct rte_dma_stats {
    uint64_t submitted;
    uint64_t completed;
    uint64_t errors;
};

#define RTE_DMA_ALL_VCHAN   0xFFFFu

int rte_dma_stats_get(int16_t dev_id, uint16_t vchan,
              struct rte_dma_stats *stats);

int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan);

enum rte_dma_vchan_status {
    RTE_DMA_VCHAN_IDLE,          
    RTE_DMA_VCHAN_ACTIVE,        
    RTE_DMA_VCHAN_HALTED_ERROR,  
};

int
rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status);

int rte_dma_dump(int16_t dev_id, FILE *f);

enum rte_dma_access_pair_group_event_type {
    RTE_DMA_GROUP_EVENT_MEMBER_LEFT,
    RTE_DMA_GROUP_EVENT_GROUP_DESTROYED
};

typedef void (*rte_dma_access_pair_group_event_cb_t)(int16_t dev_id,
    int16_t group_id,
    rte_uuid_t domain_id,
    enum rte_dma_access_pair_group_event_type event);

__rte_experimental
int rte_dma_access_pair_group_create(int16_t dev_id, rte_uuid_t domain_id, rte_uuid_t token,
                     int16_t *group_id, rte_dma_access_pair_group_event_cb_t cb);

__rte_experimental
int rte_dma_access_pair_group_destroy(int16_t dev_id, int16_t group_id);

__rte_experimental
int rte_dma_access_pair_group_join(int16_t dev_id, rte_uuid_t domain_id, rte_uuid_t token,
                   int16_t group_id, rte_dma_access_pair_group_event_cb_t cb);

__rte_experimental
int rte_dma_access_pair_group_leave(int16_t dev_id, int16_t group_id);

__rte_experimental
int rte_dma_access_pair_group_handler_get(int16_t dev_id, int16_t group_id, rte_uuid_t domain_id,
                      uint16_t *handler);

enum rte_dma_status_code {
    RTE_DMA_STATUS_SUCCESSFUL,
    RTE_DMA_STATUS_USER_ABORT,
    RTE_DMA_STATUS_NOT_ATTEMPTED,
    RTE_DMA_STATUS_INVALID_SRC_ADDR,
    RTE_DMA_STATUS_INVALID_DST_ADDR,
    RTE_DMA_STATUS_INVALID_ADDR,
    RTE_DMA_STATUS_INVALID_LENGTH,
    RTE_DMA_STATUS_INVALID_OPCODE,
    RTE_DMA_STATUS_BUS_READ_ERROR,
    RTE_DMA_STATUS_BUS_WRITE_ERROR,
    RTE_DMA_STATUS_BUS_ERROR,
    RTE_DMA_STATUS_DATA_POISION,
    RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR,
    RTE_DMA_STATUS_DEV_LINK_ERROR,
    RTE_DMA_STATUS_PAGE_FAULT,
    RTE_DMA_STATUS_ERROR_UNKNOWN = 0x100,
};

struct rte_dma_sge {
    rte_iova_t addr; 
    uint32_t length; 
};

struct rte_dma_op {
    uint64_t flags;
    struct rte_mempool *op_mp;
    enum rte_dma_status_code status;
    uint32_t rsvd;
    uint64_t impl_opaque[2];
    uint64_t user_meta;
    uint64_t event_meta;
    int16_t dma_dev_id;
    uint16_t vchan;
    uint16_t nb_src;
    uint16_t nb_dst;
    struct rte_dma_sge src_dst_seg[];
};

#ifdef __cplusplus
}
#endif

#include "rte_dmadev_core.h"
#include "rte_dmadev_trace_fp.h"

#ifdef __cplusplus
extern "C" {
#endif

#define RTE_DMA_OP_FLAG_FENCE   RTE_BIT64(0)

#define RTE_DMA_OP_FLAG_SUBMIT  RTE_BIT64(1)

#define RTE_DMA_OP_FLAG_LLC     RTE_BIT64(2)

#define RTE_DMA_OP_FLAG_AUTO_FREE   RTE_BIT64(3)

static inline int
rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
         uint32_t length, uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    int ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || length == 0)
        return -EINVAL;
    if (obj->copy == NULL)
        return -ENOTSUP;
#endif

    ret = obj->copy(obj->dev_private, vchan, src, dst, length, flags);
    rte_dma_trace_copy(dev_id, vchan, src, dst, length, flags, ret);

    return ret;
}

static inline int
rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src,
        struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,
        uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    int ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL ||
        nb_src == 0 || nb_dst == 0)
        return -EINVAL;
    if (obj->copy_sg == NULL)
        return -ENOTSUP;
#endif

    ret = obj->copy_sg(obj->dev_private, vchan, src, dst, nb_src, nb_dst, flags);
    rte_dma_trace_copy_sg(dev_id, vchan, src, dst, nb_src, nb_dst, flags,
                  ret);

    return ret;
}

static inline int
rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern,
         rte_iova_t dst, uint32_t length, uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    int ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || length == 0)
        return -EINVAL;
    if (obj->fill == NULL)
        return -ENOTSUP;
#endif

    ret = obj->fill(obj->dev_private, vchan, pattern, dst, length, flags);
    rte_dma_trace_fill(dev_id, vchan, pattern, dst, length, flags, ret);

    return ret;
}

static inline int
rte_dma_submit(int16_t dev_id, uint16_t vchan)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    int ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id))
        return -EINVAL;
    if (obj->submit == NULL)
        return -ENOTSUP;
#endif

    ret = obj->submit(obj->dev_private, vchan);
    rte_dma_trace_submit(dev_id, vchan, ret);

    return ret;
}

static inline uint16_t
rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,
          uint16_t *last_idx, bool *has_error)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t idx, ret;
    bool err;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || nb_cpls == 0)
        return 0;
    if (obj->completed == NULL)
        return 0;
#endif

    /* Ensure the pointer values are non-null to simplify drivers.
     * In most cases these should be compile time evaluated, since this is
     * an inline function.
     * - If NULL is explicitly passed as parameter, then compiler knows the
     *   value is NULL
     * - If address of local variable is passed as parameter, then compiler
     *   can know it's non-NULL.
     */
    if (last_idx == NULL)
        last_idx = &idx;
    if (has_error == NULL)
        has_error = &err;

    *has_error = false;
    ret = obj->completed(obj->dev_private, vchan, nb_cpls, last_idx, has_error);
    rte_dma_trace_completed(dev_id, vchan, nb_cpls, last_idx, has_error,
                ret);

    return ret;
}

static inline uint16_t
rte_dma_completed_status(int16_t dev_id, uint16_t vchan,
             const uint16_t nb_cpls, uint16_t *last_idx,
             enum rte_dma_status_code *status)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t idx, ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL)
        return 0;
    if (obj->completed_status == NULL)
        return 0;
#endif

    if (last_idx == NULL)
        last_idx = &idx;

    ret = obj->completed_status(obj->dev_private, vchan, nb_cpls, last_idx, status);
    rte_dma_trace_completed_status(dev_id, vchan, nb_cpls, last_idx, status,
                       ret);

    return ret;
}

static inline uint16_t
rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id))
        return 0;
    if (obj->burst_capacity == NULL)
        return 0;
#endif
    ret = obj->burst_capacity(obj->dev_private, vchan);
    rte_dma_trace_burst_capacity(dev_id, vchan, ret);

    return ret;
}

static inline uint16_t
rte_dma_enqueue_ops(int16_t dev_id, uint16_t vchan, struct rte_dma_op **ops, uint16_t nb_ops)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id))
        return 0;
    if (*obj->enqueue == NULL)
        return 0;
#endif

    ret = (*obj->enqueue)(obj->dev_private, vchan, ops, nb_ops);
    rte_dma_trace_enqueue_ops(dev_id, vchan, (void **)ops, nb_ops);

    return ret;
}

static inline uint16_t
rte_dma_dequeue_ops(int16_t dev_id, uint16_t vchan, struct rte_dma_op **ops, uint16_t nb_ops)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id))
        return 0;
    if (*obj->dequeue == NULL)
        return 0;
#endif

    ret = (*obj->dequeue)(obj->dev_private, vchan, ops, nb_ops);
    rte_dma_trace_dequeue_ops(dev_id, vchan, (void **)ops, nb_ops);

    return ret;
}

#ifdef __cplusplus
}
#endif

#endif /* RTE_DMADEV_H */