in io/iouring-wrapper.cpp [77:197]
int init(iouring_args args) {
m_args = args;
m_args.setup_sq_aff = false;
int compare_result;
if (kernel_version_compare("5.11", compare_result) == 0 && compare_result <= 0) {
rlimit resource_limit{.rlim_cur = RLIM_INFINITY, .rlim_max = RLIM_INFINITY};
if (setrlimit(RLIMIT_MEMLOCK, &resource_limit) != 0)
LOG_ERROR_RETURN(0, -1, "iouring: failed to set resource limit. "
"Use command `ulimit -l unlimited`, or change to root");
}
check_register_file_support();
check_cooperative_task_support();
set_submit_wait_function();
m_ring = new io_uring{};
io_uring_params params{};
if (m_cooperative_task_flag == 1) {
params.flags = IORING_SETUP_COOP_TASKRUN;
}
if (args.setup_iopoll)
params.flags |= IORING_SETUP_IOPOLL;
if (args.setup_sqpoll) {
params.flags |= IORING_SETUP_SQPOLL;
params.sq_thread_idle = args.sq_thread_idle_ms;
if (args.setup_sq_aff) {
params.flags |= IORING_SETUP_SQ_AFF;
params.sq_thread_cpu = args.sq_thread_cpu;
}
}
retry:
int ret = io_uring_queue_init_params(QUEUE_DEPTH, m_ring, ¶ms);
if (ret != 0) {
if (-ret == EINVAL) {
auto& p = params;
if (p.flags & IORING_SETUP_DEFER_TASKRUN) {
p.flags &= ~IORING_SETUP_DEFER_TASKRUN;
p.flags &= ~IORING_SETUP_SINGLE_ISSUER;
LOG_INFO("io_uring_queue_init failed, removing IORING_SETUP_DEFER_TASKRUN, IORING_SETUP_SINGLE_ISSUER");
goto retry;
}
if (p.flags & IORING_SETUP_COOP_TASKRUN) {
// this seems to be conflicting with IORING_SETUP_SQPOLL,
// at least in 6.4.12-1.el8.elrepo.x86_64
p.flags &= ~IORING_SETUP_COOP_TASKRUN;
LOG_INFO("io_uring_queue_init failed, removing IORING_SETUP_COOP_TASKRUN");
goto retry;
}
if (p.flags & IORING_SETUP_CQSIZE) {
p.flags &= ~IORING_SETUP_CQSIZE;
LOG_INFO("io_uring_queue_init failed, removing IORING_SETUP_CQSIZE");
goto retry;
} }
// reset m_ring so that the destructor won't do duplicate munmap cleanup (io_uring_queue_exit)
delete m_ring;
m_ring = nullptr;
LOG_ERROR_RETURN(0, -1, "iouring: failed to init queue: ", ERRNO(-ret));
}
// Check feature supported
if (!check_required_features(params, IORING_FEAT_CUR_PERSONALITY,
IORING_FEAT_NODROP, IORING_FEAT_FAST_POLL,
IORING_FEAT_EXT_ARG, IORING_FEAT_RW_CUR_POS)) {
LOG_ERROR_RETURN(0, -1, "iouring: required feature not supported");
}
// Check opcode supported
auto probe = io_uring_get_probe_ring(m_ring);
if (probe == nullptr) {
LOG_ERROR_RETURN(0, -1, "iouring: failed to get probe");
}
DEFER(io_uring_free_probe(probe));
if (!io_uring_opcode_supported(probe, IORING_OP_PROVIDE_BUFFERS) ||
!io_uring_opcode_supported(probe, IORING_OP_ASYNC_CANCEL)) {
LOG_ERROR_RETURN(0, -1, "iouring: some opcodes are not supported");
}
// Register ring fd, only available since 5.18. Doesn't have to succeed
ret = io_uring_register_ring_fd(m_ring);
if (ret < 0 && ret != -EINVAL) {
LOG_ERROR_RETURN(EINVAL, -1, "iouring: unable to register ring fd", ret);
}
// Register files. Init with sparse entries
if (register_files_enabled()) {
auto entries = new int[REGISTER_FILES_MAX_NUM];
DEFER(delete[] entries);
for (int i = 0; i < REGISTER_FILES_MAX_NUM; ++i) {
entries[i] = REGISTER_FILES_SPARSE_FD;
}
ret = io_uring_register_files(m_ring, entries, REGISTER_FILES_MAX_NUM);
if (ret != 0) {
LOG_ERROR_RETURN(-ret, -1, "iouring: unable to register files, ", ERRNO(-ret));
}
}
m_eventfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
if (m_eventfd < 0) {
LOG_ERRNO_RETURN(0, -1, "iouring: failed to create eventfd");
}
if (args.is_master) {
// Setup a multishot poll on master engine to watch the cancel_wait
uint32_t poll_mask = evmap.translate_bitwisely(EVENT_READ);
auto sqe = _get_sqe();
if (!sqe) return -1;
io_uring_prep_poll_multishot(sqe, m_eventfd, poll_mask);
io_uring_sqe_set_data(sqe, this);
ret = io_uring_submit(m_ring);
if (ret <= 0) {
LOG_ERROR_RETURN(-ret, -1, "iouring: fail to submit multishot poll, ", ERRNO(-ret));
}
} else {
// Register cascading engine to eventfd
if (io_uring_register_eventfd(m_ring, m_eventfd) != 0) {
LOG_ERRNO_RETURN(0, -1, "iouring: failed to register cascading event fd");
}
}
return 0;
}