Provided by: libcufile-dev_1.5.1.14~12.0.1-4build4_amd64 
NAME
cufile.h - cuFile C APIs
SYNOPSIS
Data Structures
struct CUfileDescr_t
struct CUfileDrvProps
struct CUfileError
cufileop status string
struct CUfileFSOps
struct CUfileIOEvents
struct CUfileIOParams
struct cufileRDMAInfo
Macros
#define CU_FILE_CUDA_ERR(status) ((status).cu_err)
#define CU_FILE_RDMA_REGISTER 1
#define CU_FILE_RDMA_RELAXED_ORDERING (1<<1)
#define CUFILE_ERRSTR(err) cufileop_status_error((CUfileOpError)abs((err)))
#define cuFileDriverClose cuFileDriverClose_v2
#define IS_CUDA_ERR(status) ((status).err == CU_FILE_CUDA_DRIVER_ERROR)
#define IS_CUFILE_ERR(err) (abs((err)) > CUFILEOP_BASE_ERR)
error macros to inspect error status of type CUfileOpError
Typedefs
typedef void * CUfileBatchHandle_t
typedef enum cufileBatchMode CUfileBatchMode_t
typedef struct CUfileDescr_t CUfileDescr_t
typedef enum CUfileDriverControlFlags CUfileDriverControlFlags_t
typedef enum CUfileDriverStatusFlags CUfileDriverStatusFlags_t
typedef struct CUfileDrvProps CUfileDrvProps_t
typedef struct CUfileError CUfileError_t
cufileop status string
typedef enum CUfileFeatureFlags CUfileFeatureFlags_t
typedef struct CUfileFSOps CUfileFSOps_t
typedef void * CUfileHandle_t
File handle type.
typedef struct CUfileIOEvents CUfileIOEvents_t
typedef struct CUfileIOParams CUfileIOParams_t
typedef enum CUfileOpcode CUfileOpcode_t
typedef struct cufileRDMAInfo cufileRDMAInfo_t
typedef enum CUFILEStatus_enum CUfileStatus_t
typedef struct sockaddr sockaddr_t
Enumerations
enum cufileBatchMode { CUFILE_BATCH = 1 }
enum CUfileDriverControlFlags { CU_FILE_USE_POLL_MODE = 0, CU_FILE_ALLOW_COMPAT_MODE = 1 }
enum CUfileDriverStatusFlags { CU_FILE_LUSTRE_SUPPORTED = 0, CU_FILE_WEKAFS_SUPPORTED = 1,
CU_FILE_NFS_SUPPORTED = 2, CU_FILE_GPFS_SUPPORTED = 3, CU_FILE_NVME_SUPPORTED = 4,
CU_FILE_NVMEOF_SUPPORTED = 5, CU_FILE_SCSI_SUPPORTED = 6, CU_FILE_SCALEFLUX_CSD_SUPPORTED = 7,
CU_FILE_NVMESH_SUPPORTED = 8, CU_FILE_BEEGFS_SUPPORTED = 9 }
enum CUfileFeatureFlags { CU_FILE_DYN_ROUTING_SUPPORTED = 0, CU_FILE_BATCH_IO_SUPPORTED = 1,
CU_FILE_STREAMS_SUPPORTED = 2 }
enum CUfileFileHandleType { CU_FILE_HANDLE_TYPE_OPAQUE_FD = 1, CU_FILE_HANDLE_TYPE_OPAQUE_WIN32 = 2,
CU_FILE_HANDLE_TYPE_USERSPACE_FS = 3 }
enum CUfileOpcode { CUFILE_READ = 0, CUFILE_WRITE }
enum CUFILEStatus_enum { CUFILE_WAITING = 0x000001, CUFILE_PENDING = 0x000002, CUFILE_INVALID = 0x000004,
CUFILE_CANCELED = 0x000008, CUFILE_COMPLETE = 0x0000010, CUFILE_TIMEOUT = 0x0000020, CUFILE_FAILED =
0x0000040 }
Functions
CUfileError_t cuFileBatchIOCancel (CUfileBatchHandle_t batch_idp)
void cuFileBatchIODestroy (CUfileBatchHandle_t batch_idp)
CUfileError_t cuFileBatchIOGetStatus (CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned *nr,
CUfileIOEvents_t *iocbp, struct timespec *timeout)
CUfileError_t cuFileBatchIOSetUp (CUfileBatchHandle_t *batch_idp, unsigned nr)
CUfileError_t cuFileBatchIOSubmit (CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t *iocbp,
unsigned int flags)
CUfileError_t cuFileBufDeregister (const void *devPtr_base)
deregister an already registered device memory from cuFile
CUfileError_t cuFileBufRegister (const void *devPtr_base, size_t length, int flags)
register an existing cudaMalloced memory with cuFile to pin for GPUDirect Storage access.
CUfileError_t cuFileDriverClose (void)
CUfileError_t cuFileDriverClose_v2 (void)
reset the cuFile library and release the nvidia-fs driver
CUfileError_t cuFileDriverGetProperties (CUfileDrvProps_t *props)
Gets the Driver session properties.
CUfileError_t cuFileDriverOpen (void)
Initialize the cuFile library and open the nvidia-fs driver.
CUfileError_t cuFileDriverSetMaxCacheSize (size_t max_cache_size)
Control parameter to set maximum GPU memory reserved per device by the library for internal
buffering.
CUfileError_t cuFileDriverSetMaxDirectIOSize (size_t max_direct_io_size)
Control parameter to set max IO size(KB) used by the library to talk to nvidia-fs driver.
CUfileError_t cuFileDriverSetMaxPinnedMemSize (size_t max_pinned_size)
Sets maximum buffer space that is pinned in KB for use by cuFileBufRegister.
CUfileError_t cuFileDriverSetPollMode (bool poll, size_t poll_threshold_size)
Sets whether the Read/Write APIs use polling to do IO operations.
void cuFileHandleDeregister (CUfileHandle_t fh)
releases a registered filehandle from cuFile
CUfileError_t cuFileHandleRegister (CUfileHandle_t *fh, CUfileDescr_t *descr)
cuFileHandleRegister is required, and performs extra checking that is memoized to provide increased
performance on later cuFile operations.
ssize_t cuFileRead (CUfileHandle_t fh, void *devPtr_base, size_t size, off_t file_offset, off_t
devPtr_offset)
read data from a registered file handle to a specified device memory
long cuFileUseCount (void)
returns use count of cufile drivers at that moment by the process.
ssize_t cuFileWrite (CUfileHandle_t fh, const void *devPtr_base, size_t size, off_t file_offset, off_t
devPtr_offset)
write data from a specified device memory to a registered file handle
Variables
CUfileOpError
Detailed Description
cuFile C APIs
This file contains all the C APIs to perform GPUDirect Storage supported IO operations
Macro Definition Documentation
#define CU_FILE_CUDA_ERR(status) ((status).cu_err)
#define CU_FILE_RDMA_REGISTER 1
#define CU_FILE_RDMA_RELAXED_ORDERING (1<<1)
#define CUFILE_ERRSTR(err) cufileop_status_error((CUfileOpError)abs((err)))
#define cuFileDriverClose cuFileDriverClose_v2
#define IS_CUDA_ERR(status) ((status).err == CU_FILE_CUDA_DRIVER_ERROR)
#define IS_CUFILE_ERR(err) (abs((err)) > CUFILEOP_BASE_ERR)
error macros to inspect error status of type CUfileOpError
Typedef Documentation
typedef void* CUfileBatchHandle_t
typedef enum cufileBatchMode CUfileBatchMode_t
typedef struct CUfileDescr_t CUfileDescr_t
typedef enum CUfileDriverControlFlags CUfileDriverControlFlags_t
typedef enum CUfileDriverStatusFlags CUfileDriverStatusFlags_t
typedef struct CUfileDrvProps CUfileDrvProps_t
typedef struct CUfileError CUfileError_t
cufileop status string
typedef enum CUfileFeatureFlags CUfileFeatureFlags_t
typedef struct CUfileFSOps CUfileFSOps_t
typedef void* CUfileHandle_t
File handle type.
typedef struct CUfileIOEvents CUfileIOEvents_t
typedef struct CUfileIOParams CUfileIOParams_t
typedef enum CUfileOpcode CUfileOpcode_t
typedef struct cufileRDMAInfo cufileRDMAInfo_t
typedef enum CUFILEStatus_enum CUfileStatus_t
typedef struct sockaddr sockaddr_t
Enumeration Type Documentation
enum cufileBatchMode
Enumerator
CUFILE_BATCH
enum CUfileDriverControlFlags
Enumerator
CU_FILE_USE_POLL_MODE
use POLL mode. properties.use_poll_mode
CU_FILE_ALLOW_COMPAT_MODE
allow COMPATIBILITY mode. properties.allow_compat_mode
enum CUfileDriverStatusFlags
Enumerator
CU_FILE_LUSTRE_SUPPORTED
Support for DDN LUSTRE
CU_FILE_WEKAFS_SUPPORTED
Support for WEKAFS
CU_FILE_NFS_SUPPORTED
Support for NFS
CU_FILE_GPFS_SUPPORTED
CU_FILE_NVME_SUPPORTED
< Support for GPFS Support for NVMe
CU_FILE_NVMEOF_SUPPORTED
Support for NVMeOF
CU_FILE_SCSI_SUPPORTED
Support for SCSI
CU_FILE_SCALEFLUX_CSD_SUPPORTED
Support for Scaleflux CSD
CU_FILE_NVMESH_SUPPORTED
Support for NVMesh Block Dev
CU_FILE_BEEGFS_SUPPORTED
Support for BeeGFS
enum CUfileFeatureFlags
Enumerator
CU_FILE_DYN_ROUTING_SUPPORTED
Support for Dynamic routing to handle devices across the PCIe bridges
CU_FILE_BATCH_IO_SUPPORTED
Unsupported
CU_FILE_STREAMS_SUPPORTED
Unsupported
enum CUfileFileHandleType
Enumerator
CU_FILE_HANDLE_TYPE_OPAQUE_FD
Linux based fd
CU_FILE_HANDLE_TYPE_OPAQUE_WIN32
Windows based handle (unsupported)
CU_FILE_HANDLE_TYPE_USERSPACE_FS
enum CUfileOpcode
Enumerator
CUFILE_READ
CUFILE_WRITE
enum CUFILEStatus_enum
Enumerator
CUFILE_WAITING
CUFILE_PENDING
CUFILE_INVALID
CUFILE_CANCELED
CUFILE_COMPLETE
CUFILE_TIMEOUT
CUFILE_FAILED
Function Documentation
CUfileError_t cuFileBatchIOCancel (CUfileBatchHandle_t batch_idp)
void cuFileBatchIODestroy (CUfileBatchHandle_t batch_idp)
CUfileError_t cuFileBatchIOGetStatus (CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned * nr,
CUfileIOEvents_t * iocbp, struct timespec * timeout)
CUfileError_t cuFileBatchIOSetUp (CUfileBatchHandle_t * batch_idp, unsigned nr)
CUfileError_t cuFileBatchIOSubmit (CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t * iocbp,
unsigned int flags)
CUfileError_t cuFileBufDeregister (const void * devPtr_base)
deregister an already registered device memory from cuFile
Parameters:
devPtr_base device pointer to deregister
Returns:
CU_FILE_SUCCESS on success
CU_FILE_INVALID_VALUE on invalid memory pointer or unregistered memory pointer
See also:
cuFileBufRegister
cuFileRead
cuFileWrite
CUfileError_t cuFileBufRegister (const void * devPtr_base, size_t length, int flags)
register an existing cudaMalloced memory with cuFile to pin for GPUDirect Storage access.
Parameters:
devPtr_base device pointer to allocated
length size of memory region from the above specified devPtr
flags CU_FILE_RDMA_REGISTER
Returns:
CU_FILE_SUCCESS on success
CU_FILE_NVFS_DRIVER_ERROR
CU_FILE_INVALID_VALUE
CU_FILE_CUDA_ERROR for unsupported memory type
CU_FILE_MEMORY_ALREADY_REGISTERED on error
CU_FILE_GPU_MEMORY_PINNING_FAILED if not enough pinned memory is available
Note:
This memory will be use to perform GPU direct DMA from the supported storage.
Warning:
This API is intended for usecases where the memory is used as streaming buffer that is reused across
multiple cuFile IO operations before calling cuFileBufDeregister
See also:
cuFileBufDeregister
cuFileRead
cuFileWrite
CUfileError_t cuFileDriverClose (void)
CUfileError_t cuFileDriverClose_v2 (void)
reset the cuFile library and release the nvidia-fs driver
Returns:
CU_FILE_SUCCESS on success
CU_FILE_DRIVER_CLOSING if there are any active IO operations using cuFileRead or cuFileWrite
See also:
cuFileDriverOpen
CUfileError_t cuFileDriverGetProperties (CUfileDrvProps_t * props)
Gets the Driver session properties.
Returns:
CU_FILE_SUCCESS on success
See also:
cuFileDriverSetPollMode
cuFileDriverSetMaxDirectIOSize
cuFileDriverSetMaxCacheSize
cuFileDriverSetMaxPinnedMemSize
CUfileError_t cuFileDriverOpen (void)
Initialize the cuFile library and open the nvidia-fs driver.
Returns:
CU_FILE_SUCCESS on success
CU_FILE_DRIVER_NOT_INITIALIZED
CU_FILE_DRIVER_VERSION_MISMATCH on driver version mismatch error
See also:
cuFileDriverClose
CUfileError_t cuFileDriverSetMaxCacheSize (size_t max_cache_size)
Control parameter to set maximum GPU memory reserved per device by the library for internal buffering.
Parameters:
max_cache_size The maximum GPU buffer space per device used for internal use in KB
Returns:
CU_FILE_SUCCESS on success
CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized
CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error
Warning:
This is an advanced command and should be tuned based on supported GPU memory
See also:
cuFileDriverGetProperties
CUfileError_t cuFileDriverSetMaxDirectIOSize (size_t max_direct_io_size)
Control parameter to set max IO size(KB) used by the library to talk to nvidia-fs driver.
Parameters:
max_direct_io_size maximum allowed direct io size in KB
Returns:
CU_FILE_SUCCESS on success
CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized
CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error
Warning:
This is an advanced command and should be tuned based on available system memory
See also:
cuFileDriverGetProperties
CUfileError_t cuFileDriverSetMaxPinnedMemSize (size_t max_pinned_size)
Sets maximum buffer space that is pinned in KB for use by cuFileBufRegister.
Parameters:
max_pinned_size maximum buffer space that is pinned in KB
Returns:
CU_FILE_SUCCESS on success
CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized
CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error
Warning:
This is an advanced command and should be tuned based on supported GPU memory
See also:
cuFileDriverGetProperties
CUfileError_t cuFileDriverSetPollMode (bool poll, size_t poll_threshold_size)
Sets whether the Read/Write APIs use polling to do IO operations.
Parameters:
poll boolean to indicate whether to use poll mode or not
poll_threshold_size max IO size to use for POLLING mode in KB
Returns:
CU_FILE_SUCCESS on success
CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized
CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error
Warning:
This is an advanced command and should be tuned based on available system memory
See also:
cuFileDriverGetProperties
void cuFileHandleDeregister (CUfileHandle_t fh)
releases a registered filehandle from cuFile
Parameters:
fh CUfileHandle_t file handle
Returns:
void
See also:
cuFileHandleRegister
CUfileError_t cuFileHandleRegister (CUfileHandle_t * fh, CUfileDescr_t * descr)
cuFileHandleRegister is required, and performs extra checking that is memoized to provide increased
performance on later cuFile operations.
Parameters:
fh CUfileHandle_t opaque file handle for IO operations
descr CUfileDescr_t file descriptor (OS agnostic)
Returns:
CU_FILE_SUCCESS on successful completion. fh will be updated for use in cuFileRead, cuFileWrite,
cuFileHandleDeregister
CU_FILE_DRIVER_NOT_INITIALIZED on failure to load driver
CU_FILE_IO_NOT_SUPPORTED - if filesystem is not supported
CU_FILE_INVALID_VALUE if null or bad api arguments
CU_FILE_INVALID_FILE_OPEN_FLAG if file is opened with unsupported modes like no O_DIRECT
CU_FILE_INVALID_FILE_TYPE if filepath is not valid or is not a regular file
CU_FILE_HANDLE_ALREADY_REGISTERED if file handle/descriptor is already registered
Description cuFileHandleRegister registers the open file descriptor for use with cuFile IO operations.
This API will ensure that the file’s descriptor is checked for GPUDirect Storage support and returns a
valid file handle on CU_FILE_SUCCESS.
Note:
the file needs to be opened in O_DIRECT mode to support GPUDirect Storage.
See also:
cuFileRead
cuFileWrite
cuFileHandleDeregister
ssize_t cuFileRead (CUfileHandle_t fh, void * devPtr_base, size_t size, off_t file_offset, off_t
devPtr_offset)
read data from a registered file handle to a specified device memory
Parameters:
fh CUfileHandle_t opaque file handle
devPtr_base base address of buffer in device memory
size size bytes to read
file_offset file-offset from beginning of the file
devPtr_offset offset relative to the devPtr_base pointer to read into.
Returns:
size of bytes successfully read
-1 on error, in which case errno is set to indicate filesystem errors.
all other errors will return a negative integer value of CUfileOpError enum value.
Note:
If the devPtr is not registered with cuFileBufRegister, the data will be buffered through
preallocated pinned buffers.
This is useful for applications that need to perform IO to unaligned file offsets and/or size. This
is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated
memory.
See also:
cuFileBufRegister
cuFileHandleRegister
cuFileWrite
long cuFileUseCount (void)
returns use count of cufile drivers at that moment by the process.
ssize_t cuFileWrite (CUfileHandle_t fh, const void * devPtr_base, size_t size, off_t file_offset, off_t
devPtr_offset)
write data from a specified device memory to a registered file handle
Parameters:
fh CUfileHandle_t opaque file handle
devPtr_base base address of buffer in device memory
size size bytes to write
file_offset file-offset from beginning of the file
devPtr_offset offset relative to the devPtr_base pointer to read into.
Returns:
size of bytes successfully written
-1 on error, in which case errno is set to indicate filesystem errors.
all other errors will return a negative integer value of CUfileOpError enum value.
Note:
If the devPtr is not registered with cuFileBufRegister, the data will be buffered through
preallocated pinned buffers.
This is useful for applications that need to perform IO to unaligned file offsets and/or size. This
is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated
memory.
See also:
cuFileBufRegister
cuFileHandleRegister
cuFileRead
Variable Documentation
CUfileOpError
Author
Generated automatically by Doxygen for GPUDirect Storage from the source code.
GPUDirect Storage Wed Dec 14 2022 cufile.h(3)