in dmatest.c [568:933]
static int dmatest_func(void *data)
{
struct dmatest_thread *thread = data;
struct dmatest_done *done = &thread->test_done;
struct dmatest_info *info;
struct dmatest_params *params;
struct dma_chan *chan;
struct dma_device *dev;
struct device *dma_dev;
unsigned int error_count;
unsigned int failed_tests = 0;
unsigned int total_tests = 0;
dma_cookie_t cookie;
enum dma_status status;
enum dma_ctrl_flags flags;
u8 *pq_coefs = NULL;
int ret;
unsigned int buf_size;
struct dmatest_data *src;
struct dmatest_data *dst;
int i;
ktime_t ktime, start, diff;
ktime_t filltime = 0;
ktime_t comparetime = 0;
s64 runtime = 0;
unsigned long long total_len = 0;
unsigned long long iops = 0;
u8 align = 0;
bool is_memset = false;
dma_addr_t *srcs;
dma_addr_t *dma_pq;
set_freezable();
ret = -ENOMEM;
smp_rmb();
thread->pending = false;
info = thread->info;
params = &info->params;
chan = thread->chan;
dev = chan->device;
dma_dev = dmaengine_get_dma_device(chan);
src = &thread->src;
dst = &thread->dst;
if (thread->type == DMA_MEMCPY) {
align = params->alignment < 0 ? dev->copy_align :
params->alignment;
src->cnt = dst->cnt = 1;
} else if (thread->type == DMA_MEMSET) {
align = params->alignment < 0 ? dev->fill_align :
params->alignment;
src->cnt = dst->cnt = 1;
is_memset = true;
} else if (thread->type == DMA_XOR) {
/* force odd to ensure dst = src */
src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);
dst->cnt = 1;
align = params->alignment < 0 ? dev->xor_align :
params->alignment;
} else if (thread->type == DMA_PQ) {
/* force odd to ensure dst = src */
src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
dst->cnt = 2;
align = params->alignment < 0 ? dev->pq_align :
params->alignment;
pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
if (!pq_coefs)
goto err_thread_type;
for (i = 0; i < src->cnt; i++)
pq_coefs[i] = 1;
} else
goto err_thread_type;
/* Check if buffer count fits into map count variable (u8) */
if ((src->cnt + dst->cnt) >= 255) {
pr_err("too many buffers (%d of 255 supported)\n",
src->cnt + dst->cnt);
goto err_free_coefs;
}
buf_size = params->buf_size;
if (1 << align > buf_size) {
pr_err("%u-byte buffer too small for %d-byte alignment\n",
buf_size, 1 << align);
goto err_free_coefs;
}
if (dmatest_alloc_test_data(src, buf_size, align) < 0)
goto err_free_coefs;
if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
goto err_src;
set_user_nice(current, 10);
srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);
if (!srcs)
goto err_dst;
dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);
if (!dma_pq)
goto err_srcs_array;
/*
* src and dst buffers are freed by ourselves below
*/
if (params->polled)
flags = DMA_CTRL_ACK;
else
flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
ktime = ktime_get();
while (!(kthread_should_stop() ||
(params->iterations && total_tests >= params->iterations))) {
struct dma_async_tx_descriptor *tx = NULL;
struct dmaengine_unmap_data *um;
dma_addr_t *dsts;
unsigned int len;
total_tests++;
if (params->transfer_size) {
if (params->transfer_size >= buf_size) {
pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
params->transfer_size, buf_size);
break;
}
len = params->transfer_size;
} else if (params->norandom) {
len = buf_size;
} else {
len = dmatest_random() % buf_size + 1;
}
/* Do not alter transfer size explicitly defined by user */
if (!params->transfer_size) {
len = (len >> align) << align;
if (!len)
len = 1 << align;
}
total_len += len;
if (params->norandom) {
src->off = 0;
dst->off = 0;
} else {
src->off = dmatest_random() % (buf_size - len + 1);
dst->off = dmatest_random() % (buf_size - len + 1);
src->off = (src->off >> align) << align;
dst->off = (dst->off >> align) << align;
}
if (!params->noverify) {
start = ktime_get();
dmatest_init_srcs(src->aligned, src->off, len,
buf_size, is_memset);
dmatest_init_dsts(dst->aligned, dst->off, len,
buf_size, is_memset);
diff = ktime_sub(ktime_get(), start);
filltime = ktime_add(filltime, diff);
}
um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt,
GFP_KERNEL);
if (!um) {
failed_tests++;
result("unmap data NULL", total_tests,
src->off, dst->off, len, ret);
continue;
}
um->len = buf_size;
for (i = 0; i < src->cnt; i++) {
void *buf = src->aligned[i];
struct page *pg = virt_to_page(buf);
unsigned long pg_off = offset_in_page(buf);
um->addr[i] = dma_map_page(dma_dev, pg, pg_off,
um->len, DMA_TO_DEVICE);
srcs[i] = um->addr[i] + src->off;
ret = dma_mapping_error(dma_dev, um->addr[i]);
if (ret) {
result("src mapping error", total_tests,
src->off, dst->off, len, ret);
goto error_unmap_continue;
}
um->to_cnt++;
}
/* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
dsts = &um->addr[src->cnt];
for (i = 0; i < dst->cnt; i++) {
void *buf = dst->aligned[i];
struct page *pg = virt_to_page(buf);
unsigned long pg_off = offset_in_page(buf);
dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len,
DMA_BIDIRECTIONAL);
ret = dma_mapping_error(dma_dev, dsts[i]);
if (ret) {
result("dst mapping error", total_tests,
src->off, dst->off, len, ret);
goto error_unmap_continue;
}
um->bidi_cnt++;
}
if (thread->type == DMA_MEMCPY)
tx = dev->device_prep_dma_memcpy(chan,
dsts[0] + dst->off,
srcs[0], len, flags);
else if (thread->type == DMA_MEMSET)
tx = dev->device_prep_dma_memset(chan,
dsts[0] + dst->off,
*(src->aligned[0] + src->off),
len, flags);
else if (thread->type == DMA_XOR)
tx = dev->device_prep_dma_xor(chan,
dsts[0] + dst->off,
srcs, src->cnt,
len, flags);
else if (thread->type == DMA_PQ) {
for (i = 0; i < dst->cnt; i++)
dma_pq[i] = dsts[i] + dst->off;
tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
src->cnt, pq_coefs,
len, flags);
}
if (!tx) {
result("prep error", total_tests, src->off,
dst->off, len, ret);
msleep(100);
goto error_unmap_continue;
}
done->done = false;
if (!params->polled) {
tx->callback = dmatest_callback;
tx->callback_param = done;
}
cookie = tx->tx_submit(tx);
if (dma_submit_error(cookie)) {
result("submit error", total_tests, src->off,
dst->off, len, ret);
msleep(100);
goto error_unmap_continue;
}
if (params->polled) {
status = dma_sync_wait(chan, cookie);
dmaengine_terminate_sync(chan);
if (status == DMA_COMPLETE)
done->done = true;
} else {
dma_async_issue_pending(chan);
wait_event_freezable_timeout(thread->done_wait,
done->done,
msecs_to_jiffies(params->timeout));
status = dma_async_is_tx_complete(chan, cookie, NULL,
NULL);
}
if (!done->done) {
result("test timed out", total_tests, src->off, dst->off,
len, 0);
goto error_unmap_continue;
} else if (status != DMA_COMPLETE &&
!(dma_has_cap(DMA_COMPLETION_NO_ORDER,
dev->cap_mask) &&
status == DMA_OUT_OF_ORDER)) {
result(status == DMA_ERROR ?
"completion error status" :
"completion busy status", total_tests, src->off,
dst->off, len, ret);
goto error_unmap_continue;
}
dmaengine_unmap_put(um);
if (params->noverify) {
verbose_result("test passed", total_tests, src->off,
dst->off, len, 0);
continue;
}
start = ktime_get();
pr_debug("%s: verifying source buffer...\n", current->comm);
error_count = dmatest_verify(src->aligned, 0, src->off,
0, PATTERN_SRC, true, is_memset);
error_count += dmatest_verify(src->aligned, src->off,
src->off + len, src->off,
PATTERN_SRC | PATTERN_COPY, true, is_memset);
error_count += dmatest_verify(src->aligned, src->off + len,
buf_size, src->off + len,
PATTERN_SRC, true, is_memset);
pr_debug("%s: verifying dest buffer...\n", current->comm);
error_count += dmatest_verify(dst->aligned, 0, dst->off,
0, PATTERN_DST, false, is_memset);
error_count += dmatest_verify(dst->aligned, dst->off,
dst->off + len, src->off,
PATTERN_SRC | PATTERN_COPY, false, is_memset);
error_count += dmatest_verify(dst->aligned, dst->off + len,
buf_size, dst->off + len,
PATTERN_DST, false, is_memset);
diff = ktime_sub(ktime_get(), start);
comparetime = ktime_add(comparetime, diff);
if (error_count) {
result("data error", total_tests, src->off, dst->off,
len, error_count);
failed_tests++;
} else {
verbose_result("test passed", total_tests, src->off,
dst->off, len, 0);
}
continue;
error_unmap_continue:
dmaengine_unmap_put(um);
failed_tests++;
}
ktime = ktime_sub(ktime_get(), ktime);
ktime = ktime_sub(ktime, comparetime);
ktime = ktime_sub(ktime, filltime);
runtime = ktime_to_us(ktime);
ret = 0;
kfree(dma_pq);
err_srcs_array:
kfree(srcs);
err_dst:
dmatest_free_test_data(dst);
err_src:
dmatest_free_test_data(src);
err_free_coefs:
kfree(pq_coefs);
err_thread_type:
iops = dmatest_persec(runtime, total_tests);
pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
current->comm, total_tests, failed_tests,
FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
dmatest_KBs(runtime, total_len), ret);
/* terminate all transfers on specified channels */
if (ret || failed_tests)
dmaengine_terminate_sync(chan);
thread->done = true;
wake_up(&thread_wait);
return ret;
}