rte_mbuf_core.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation.
 * Copyright 2014 6WIND S.A.
 */

#ifndef _RTE_MBUF_CORE_H_
#define _RTE_MBUF_CORE_H_

#include <stdalign.h>
#include <stdint.h>

#include <rte_byteorder.h>
#include <rte_stdatomic.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Packet Offload Features Flags. It also carry packet type information.
 * Critical resources. Both rx/tx shared these bits. Be cautious on any change
 *
 * - RX flags start at bit position zero, and get added to the left of previous
 *   flags.
 * - The most-significant 3 bits are reserved for generic mbuf flags
 * - TX flags therefore start at bit position 60 (i.e. 63-3), and new flags get
 *   added to the right of the previously defined flags i.e. they should count
 *   downwards, not upwards.
 *
 * Keep these flags synchronized with rte_get_rx_ol_flag_name(), rte_get_rx_ol_flag_list(),
 * rte_get_tx_ol_flag_name(), and rte_get_tx_ol_flag_list().
 */

#define RTE_MBUF_F_RX_VLAN          (1ULL << 0)

#define RTE_MBUF_F_RX_RSS_HASH      (1ULL << 1)

#define RTE_MBUF_F_RX_FDIR          (1ULL << 2)

#define RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD (1ULL << 5)

#define RTE_MBUF_F_RX_VLAN_STRIPPED (1ULL << 6)

#define RTE_MBUF_F_RX_IP_CKSUM_MASK ((1ULL << 4) | (1ULL << 7))

#define RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN 0
#define RTE_MBUF_F_RX_IP_CKSUM_BAD     (1ULL << 4)
#define RTE_MBUF_F_RX_IP_CKSUM_GOOD    (1ULL << 7)
#define RTE_MBUF_F_RX_IP_CKSUM_NONE    ((1ULL << 4) | (1ULL << 7))

#define RTE_MBUF_F_RX_L4_CKSUM_MASK ((1ULL << 3) | (1ULL << 8))

#define RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN 0
#define RTE_MBUF_F_RX_L4_CKSUM_BAD     (1ULL << 3)
#define RTE_MBUF_F_RX_L4_CKSUM_GOOD    (1ULL << 8)
#define RTE_MBUF_F_RX_L4_CKSUM_NONE    ((1ULL << 3) | (1ULL << 8))

#define RTE_MBUF_F_RX_IEEE1588_PTP  (1ULL << 9)

#define RTE_MBUF_F_RX_IEEE1588_TMST (1ULL << 10)

#define RTE_MBUF_F_RX_FDIR_ID       (1ULL << 13)

#define RTE_MBUF_F_RX_FDIR_FLX      (1ULL << 14)

#define RTE_MBUF_F_RX_QINQ_STRIPPED (1ULL << 15)

#define RTE_MBUF_F_RX_LRO           (1ULL << 16)

/* There is no flag defined at offset 17. It is free for any future use. */

#define RTE_MBUF_F_RX_SEC_OFFLOAD   (1ULL << 18)

#define RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED    (1ULL << 19)

#define RTE_MBUF_F_RX_QINQ          (1ULL << 20)

#define RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK   ((1ULL << 21) | (1ULL << 22))

#define RTE_MBUF_F_RX_OUTER_L4_CKSUM_UNKNOWN    0
#define RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD    (1ULL << 21)
#define RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD   (1ULL << 22)
#define RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID    ((1ULL << 21) | (1ULL << 22))

/*
 * Add new Rx flags here.
 * Don't forget to update RTE_MBUF_F_FIRST_FREE,
 * rte_get_rx_ol_flag_name(), and rte_get_rx_ol_flag_list().
 */

#define RTE_MBUF_F_FIRST_FREE (1ULL << 23)
#define RTE_MBUF_F_LAST_FREE (1ULL << 40)

/*
 * Add new Tx flags here.
 * Don't forget to update RTE_MBUF_F_LAST_FREE, RTE_MBUF_F_TX_OFFLOAD_MASK,
 * rte_get_tx_ol_flag_name(), and rte_get_tx_ol_flag_list().
 */

#define RTE_MBUF_F_TX_OUTER_UDP_CKSUM     (1ULL << 41)

#define RTE_MBUF_F_TX_UDP_SEG   (1ULL << 42)

#define RTE_MBUF_F_TX_SEC_OFFLOAD   (1ULL << 43)

#define RTE_MBUF_F_TX_MACSEC        (1ULL << 44)

#define RTE_MBUF_F_TX_TUNNEL_VXLAN   (0x1ULL << 45)
#define RTE_MBUF_F_TX_TUNNEL_GRE     (0x2ULL << 45)
#define RTE_MBUF_F_TX_TUNNEL_IPIP    (0x3ULL << 45)
#define RTE_MBUF_F_TX_TUNNEL_GENEVE  (0x4ULL << 45)

#define RTE_MBUF_F_TX_TUNNEL_MPLSINUDP (0x5ULL << 45)
#define RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE (0x6ULL << 45)
#define RTE_MBUF_F_TX_TUNNEL_GTP       (0x7ULL << 45)
#define RTE_MBUF_F_TX_TUNNEL_ESP       (0x8ULL << 45)

#define RTE_MBUF_F_TX_TUNNEL_IP (0xDULL << 45)

#define RTE_MBUF_F_TX_TUNNEL_UDP (0xEULL << 45)
/* add new TX TUNNEL type here */
#define RTE_MBUF_F_TX_TUNNEL_MASK    (0xFULL << 45)

#define RTE_MBUF_F_TX_QINQ        (1ULL << 49)

#define RTE_MBUF_F_TX_TCP_SEG       (1ULL << 50)

#define RTE_MBUF_F_TX_IEEE1588_TMST (1ULL << 51)

/*
 * Bits 52+53 used for L4 packet type with checksum enabled: 00: Reserved,
 * 01: TCP checksum, 10: SCTP checksum, 11: UDP checksum. To use hardware
 * L4 checksum offload, the user needs to:
 *  - fill l2_len and l3_len in mbuf
 *  - set the flags RTE_MBUF_F_TX_TCP_CKSUM, RTE_MBUF_F_TX_SCTP_CKSUM or
 *    RTE_MBUF_F_TX_UDP_CKSUM
 *  - set the flag RTE_MBUF_F_TX_IPV4 or RTE_MBUF_F_TX_IPV6
 */

#define RTE_MBUF_F_TX_L4_NO_CKSUM   (0ULL << 52)

#define RTE_MBUF_F_TX_TCP_CKSUM     (1ULL << 52)

#define RTE_MBUF_F_TX_SCTP_CKSUM    (2ULL << 52)

#define RTE_MBUF_F_TX_UDP_CKSUM     (3ULL << 52)

#define RTE_MBUF_F_TX_L4_MASK       (3ULL << 52)

#define RTE_MBUF_F_TX_IP_CKSUM      (1ULL << 54)

#define RTE_MBUF_F_TX_IPV4          (1ULL << 55)

#define RTE_MBUF_F_TX_IPV6          (1ULL << 56)

#define RTE_MBUF_F_TX_VLAN          (1ULL << 57)

#define RTE_MBUF_F_TX_OUTER_IP_CKSUM   (1ULL << 58)

#define RTE_MBUF_F_TX_OUTER_IPV4   (1ULL << 59)

#define RTE_MBUF_F_TX_OUTER_IPV6    (1ULL << 60)

#define RTE_MBUF_F_TX_OFFLOAD_MASK (    \
        RTE_MBUF_F_TX_OUTER_IPV6 |   \
        RTE_MBUF_F_TX_OUTER_IPV4 |   \
        RTE_MBUF_F_TX_OUTER_IP_CKSUM |  \
        RTE_MBUF_F_TX_VLAN |        \
        RTE_MBUF_F_TX_IPV6 |         \
        RTE_MBUF_F_TX_IPV4 |         \
        RTE_MBUF_F_TX_IP_CKSUM |        \
        RTE_MBUF_F_TX_L4_MASK |         \
        RTE_MBUF_F_TX_IEEE1588_TMST |    \
        RTE_MBUF_F_TX_TCP_SEG |         \
        RTE_MBUF_F_TX_QINQ |        \
        RTE_MBUF_F_TX_TUNNEL_MASK |  \
        RTE_MBUF_F_TX_MACSEC |       \
        RTE_MBUF_F_TX_SEC_OFFLOAD |  \
        RTE_MBUF_F_TX_UDP_SEG |  \
        RTE_MBUF_F_TX_OUTER_UDP_CKSUM)

#define RTE_MBUF_F_EXTERNAL    (1ULL << 61)

#define RTE_MBUF_F_INDIRECT    (1ULL << 62) 
#define RTE_MBUF_PRIV_ALIGN 8

#define RTE_MBUF_DEFAULT_DATAROOM   2048
#define RTE_MBUF_DEFAULT_BUF_SIZE   \
    (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM)

struct rte_mbuf_sched {
    uint32_t queue_id;   
    uint8_t traffic_class;
    uint8_t color;
    uint16_t reserved;   
}; 
enum {
    RTE_MBUF_L2_LEN_BITS = 7,
    RTE_MBUF_L3_LEN_BITS = 9,
    RTE_MBUF_L4_LEN_BITS = 8,
    RTE_MBUF_TSO_SEGSZ_BITS = 16,
    RTE_MBUF_OUTL3_LEN_BITS = 9,
    RTE_MBUF_OUTL2_LEN_BITS = 7,
    RTE_MBUF_TXOFLD_UNUSED_BITS = sizeof(uint64_t) * CHAR_BIT -
        RTE_MBUF_L2_LEN_BITS -
        RTE_MBUF_L3_LEN_BITS -
        RTE_MBUF_L4_LEN_BITS -
        RTE_MBUF_TSO_SEGSZ_BITS -
        RTE_MBUF_OUTL3_LEN_BITS -
        RTE_MBUF_OUTL2_LEN_BITS,
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
    RTE_MBUF_L2_LEN_OFS =
        sizeof(uint64_t) * CHAR_BIT - RTE_MBUF_L2_LEN_BITS,
    RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS - RTE_MBUF_L3_LEN_BITS,
    RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS - RTE_MBUF_L4_LEN_BITS,
    RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS - RTE_MBUF_TSO_SEGSZ_BITS,
    RTE_MBUF_OUTL3_LEN_OFS =
        RTE_MBUF_TSO_SEGSZ_OFS - RTE_MBUF_OUTL3_LEN_BITS,
    RTE_MBUF_OUTL2_LEN_OFS =
        RTE_MBUF_OUTL3_LEN_OFS - RTE_MBUF_OUTL2_LEN_BITS,
    RTE_MBUF_TXOFLD_UNUSED_OFS =
        RTE_MBUF_OUTL2_LEN_OFS - RTE_MBUF_TXOFLD_UNUSED_BITS,
#else
    RTE_MBUF_L2_LEN_OFS = 0,
    RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS + RTE_MBUF_L2_LEN_BITS,
    RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS + RTE_MBUF_L3_LEN_BITS,
    RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS + RTE_MBUF_L4_LEN_BITS,
    RTE_MBUF_OUTL3_LEN_OFS =
        RTE_MBUF_TSO_SEGSZ_OFS + RTE_MBUF_TSO_SEGSZ_BITS,
    RTE_MBUF_OUTL2_LEN_OFS =
        RTE_MBUF_OUTL3_LEN_OFS + RTE_MBUF_OUTL3_LEN_BITS,
    RTE_MBUF_TXOFLD_UNUSED_OFS =
        RTE_MBUF_OUTL2_LEN_OFS + RTE_MBUF_OUTL2_LEN_BITS,
#endif
};

struct __rte_cache_aligned rte_mbuf {
    void *buf_addr;           
#if RTE_IOVA_IN_MBUF

    alignas(sizeof(rte_iova_t)) rte_iova_t buf_iova;
#else

    struct rte_mbuf *next;
#endif

    /*
     * Next 8 bytes are initialised on Rx descriptor rearm,
     * or on Rx when pulling packet from descriptor.
     */
    union {
        uint64_t rearm_data[1];
        __extension__
        struct {
            uint16_t data_off;

            RTE_ATOMIC(uint16_t) refcnt;

            uint16_t nb_segs;

            uint16_t port;
        };
    };

    uint64_t ol_flags;        
    /* Remaining 24 bytes are set on Rx when pulling packet from descriptor. */
    union {
        /* void * type of the array elements is retained for driver compatibility. */
        void *rx_descriptor_fields1[24 / sizeof(void *)];
        __extension__
        struct {
            /*
             * The packet type, which is the combination of outer/inner L2, L3, L4
             * and tunnel types. The packet_type is about data really present in the
             * mbuf. Example: if vlan stripping is enabled, a received vlan packet
             * would have RTE_PTYPE_L2_ETHER and not RTE_PTYPE_L2_VLAN because the
             * vlan is stripped from the data.
             */
            union {
                uint32_t packet_type; 
                __extension__
                struct {
                    uint8_t l2_type:4;   
                    uint8_t l3_type:4;   
                    uint8_t l4_type:4;   
                    uint8_t tun_type:4;  
                    union {
                        uint8_t inner_esp_next_proto;
                        __extension__
                        struct {
                            uint8_t inner_l2_type:4;
                            uint8_t inner_l3_type:4;
                        };
                    };
                    uint8_t inner_l4_type:4; 
                };
            };

            uint32_t pkt_len;         
            uint16_t data_len;        
            uint16_t vlan_tci;

            union {
                union {
                    uint32_t rss;     
                    struct {
                        union {
                            struct {
                                uint16_t hash;
                                uint16_t id;
                            };
                            uint32_t lo;
                        };
                        uint32_t hi;
                    } fdir; 
                    struct rte_mbuf_sched sched;
                    struct {
                        uint32_t reserved1;
                        uint16_t reserved2;
                        uint16_t txq;
                    } txadapter; 
                    uint32_t usr;
                } hash;                   
            };

            uint16_t vlan_tci_outer;

            uint16_t buf_len;         
        };
    };

    struct rte_mempool *pool; 
    /*
     * Second cache line - fields only used in slow path or on Tx.
     * In special cases, some of these fields are also set on Rx,
     * most notably the 'next' field is set on Rx scattered packets.
     */
#if RTE_IOVA_IN_MBUF

    alignas(RTE_CACHE_LINE_MIN_SIZE)
    struct rte_mbuf *next;
#else

    alignas(RTE_CACHE_LINE_MIN_SIZE)
    uint64_t dynfield2;
#endif

    /* fields to support TX offloads */
    union {
        uint64_t tx_offload;       
        __extension__
        struct {
            uint64_t l2_len:RTE_MBUF_L2_LEN_BITS;
            uint64_t l3_len:RTE_MBUF_L3_LEN_BITS;
            uint64_t l4_len:RTE_MBUF_L4_LEN_BITS;
            uint64_t tso_segsz:RTE_MBUF_TSO_SEGSZ_BITS;
            /*
             * Fields for Tx offloading of tunnels.
             * These are undefined for packets which don't request
             * any tunnel offloads (outer IP or UDP checksum,
             * tunnel TSO).
             *
             * PMDs should not use these fields unconditionally
             * when calculating offsets.
             *
             * Applications are expected to set appropriate tunnel
             * offload flags when they fill in these fields.
             */
            uint64_t outer_l3_len:RTE_MBUF_OUTL3_LEN_BITS;
            uint64_t outer_l2_len:RTE_MBUF_OUTL2_LEN_BITS;
            /* uint64_t unused:RTE_MBUF_TXOFLD_UNUSED_BITS; */
        };
    };

    struct rte_mbuf_ext_shared_info *shinfo;

    uint16_t priv_size;

    uint16_t timesync;

    uint32_t dynfield1[9]; 
};

typedef void (*rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque);

struct rte_mbuf_ext_shared_info {
    rte_mbuf_extbuf_free_callback_t free_cb; 
    void *fcb_opaque;                        
    RTE_ATOMIC(uint16_t) refcnt;
};

#define RTE_MBUF_MAX_NB_SEGS    UINT16_MAX

#define RTE_MBUF_CLONED(mb)     ((mb)->ol_flags & RTE_MBUF_F_INDIRECT)

#define RTE_MBUF_HAS_EXTBUF(mb) ((mb)->ol_flags & RTE_MBUF_F_EXTERNAL)

#ifdef __DOXYGEN__
#define RTE_MBUF_DIRECT(mb) \
    !(((const char *)(&(mb)->ol_flags))[MSB_OFFSET /* 7 or 0, depending on endianness */] & \
    (char)((RTE_MBUF_F_INDIRECT | RTE_MBUF_F_EXTERNAL) >> (7 * CHAR_BIT)) /* 0x60 */)
#else /* !__DOXYGEN__ */
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
/* On little endian architecture, the MSB of a 64-bit integer is at byte offset 7. */
#define RTE_MBUF_DIRECT(mb) !(((const char *)(&(mb)->ol_flags))[7] & 0x60)
#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN
/* On big endian architecture, the MSB of a 64-bit integer is at byte offset 0. */
#define RTE_MBUF_DIRECT(mb) !(((const char *)(&(mb)->ol_flags))[0] & 0x60)
#endif /* RTE_BYTE_ORDER */
#endif /* !__DOXYGEN__ */
/* Verify the optimization above. */
static_assert((RTE_MBUF_F_INDIRECT | RTE_MBUF_F_EXTERNAL) == UINT64_C(0x60) << (7 * CHAR_BIT),
        "(RTE_MBUF_F_INDIRECT | RTE_MBUF_F_EXTERNAL) is not 0x60 at MSB");

#define RTE_MBUF_PORT_INVALID UINT16_MAX

#define MBUF_INVALID_PORT RTE_MBUF_PORT_INVALID

#define rte_pktmbuf_mtod_offset(m, t, o)    \
    ((t)(void *)((char *)(m)->buf_addr + (m)->data_off + (o)))

#define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0)

#define rte_pktmbuf_iova_offset(m, o) \
    (rte_iova_t)(rte_mbuf_iova_get(m) + (m)->data_off + (o))

#define rte_pktmbuf_iova(m) rte_pktmbuf_iova_offset(m, 0)

#ifdef __cplusplus
}
#endif

#endif /* _RTE_MBUF_CORE_H_ */