in dm-stats.c [234:391]
static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end,
sector_t step, unsigned stat_flags,
unsigned n_histogram_entries,
unsigned long long *histogram_boundaries,
const char *program_id, const char *aux_data,
void (*suspend_callback)(struct mapped_device *),
void (*resume_callback)(struct mapped_device *),
struct mapped_device *md)
{
struct list_head *l;
struct dm_stat *s, *tmp_s;
sector_t n_entries;
size_t ni;
size_t shared_alloc_size;
size_t percpu_alloc_size;
size_t histogram_alloc_size;
struct dm_stat_percpu *p;
int cpu;
int ret_id;
int r;
if (end < start || !step)
return -EINVAL;
n_entries = end - start;
if (dm_sector_div64(n_entries, step))
n_entries++;
if (n_entries != (size_t)n_entries || !(size_t)(n_entries + 1))
return -EOVERFLOW;
shared_alloc_size = struct_size(s, stat_shared, n_entries);
if ((shared_alloc_size - sizeof(struct dm_stat)) / sizeof(struct dm_stat_shared) != n_entries)
return -EOVERFLOW;
percpu_alloc_size = (size_t)n_entries * sizeof(struct dm_stat_percpu);
if (percpu_alloc_size / sizeof(struct dm_stat_percpu) != n_entries)
return -EOVERFLOW;
histogram_alloc_size = (n_histogram_entries + 1) * (size_t)n_entries * sizeof(unsigned long long);
if (histogram_alloc_size / (n_histogram_entries + 1) != (size_t)n_entries * sizeof(unsigned long long))
return -EOVERFLOW;
if (!check_shared_memory(shared_alloc_size + histogram_alloc_size +
num_possible_cpus() * (percpu_alloc_size + histogram_alloc_size)))
return -ENOMEM;
s = dm_kvzalloc(shared_alloc_size, NUMA_NO_NODE);
if (!s)
return -ENOMEM;
s->stat_flags = stat_flags;
s->n_entries = n_entries;
s->start = start;
s->end = end;
s->step = step;
s->shared_alloc_size = shared_alloc_size;
s->percpu_alloc_size = percpu_alloc_size;
s->histogram_alloc_size = histogram_alloc_size;
s->n_histogram_entries = n_histogram_entries;
s->histogram_boundaries = kmemdup(histogram_boundaries,
s->n_histogram_entries * sizeof(unsigned long long), GFP_KERNEL);
if (!s->histogram_boundaries) {
r = -ENOMEM;
goto out;
}
s->program_id = kstrdup(program_id, GFP_KERNEL);
if (!s->program_id) {
r = -ENOMEM;
goto out;
}
s->aux_data = kstrdup(aux_data, GFP_KERNEL);
if (!s->aux_data) {
r = -ENOMEM;
goto out;
}
for (ni = 0; ni < n_entries; ni++) {
atomic_set(&s->stat_shared[ni].in_flight[READ], 0);
atomic_set(&s->stat_shared[ni].in_flight[WRITE], 0);
}
if (s->n_histogram_entries) {
unsigned long long *hi;
hi = dm_kvzalloc(s->histogram_alloc_size, NUMA_NO_NODE);
if (!hi) {
r = -ENOMEM;
goto out;
}
for (ni = 0; ni < n_entries; ni++) {
s->stat_shared[ni].tmp.histogram = hi;
hi += s->n_histogram_entries + 1;
}
}
for_each_possible_cpu(cpu) {
p = dm_kvzalloc(percpu_alloc_size, cpu_to_node(cpu));
if (!p) {
r = -ENOMEM;
goto out;
}
s->stat_percpu[cpu] = p;
if (s->n_histogram_entries) {
unsigned long long *hi;
hi = dm_kvzalloc(s->histogram_alloc_size, cpu_to_node(cpu));
if (!hi) {
r = -ENOMEM;
goto out;
}
for (ni = 0; ni < n_entries; ni++) {
p[ni].histogram = hi;
hi += s->n_histogram_entries + 1;
}
}
}
/*
* Suspend/resume to make sure there is no i/o in flight,
* so that newly created statistics will be exact.
*
* (note: we couldn't suspend earlier because we must not
* allocate memory while suspended)
*/
suspend_callback(md);
mutex_lock(&stats->mutex);
s->id = 0;
list_for_each(l, &stats->list) {
tmp_s = container_of(l, struct dm_stat, list_entry);
if (WARN_ON(tmp_s->id < s->id)) {
r = -EINVAL;
goto out_unlock_resume;
}
if (tmp_s->id > s->id)
break;
if (unlikely(s->id == INT_MAX)) {
r = -ENFILE;
goto out_unlock_resume;
}
s->id++;
}
ret_id = s->id;
list_add_tail_rcu(&s->list_entry, l);
mutex_unlock(&stats->mutex);
resume_callback(md);
return ret_id;
out_unlock_resume:
mutex_unlock(&stats->mutex);
resume_callback(md);
out:
dm_stat_free(&s->rcu_head);
return r;
}