in drbd/drbd_bitmap.c [1031:1179]
static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned lazy_writeout_upper_idx) __must_hold(local)
{
struct drbd_bm_aio_ctx *ctx;
struct drbd_bitmap *b = device->bitmap;
unsigned int num_pages, i, count = 0;
unsigned long now;
char ppb[10];
int err = 0;
/*
* We are protected against bitmap disappearing/resizing by holding an
* ldev reference (caller must have called get_ldev()).
* For read/write, we are protected against changes to the bitmap by
* the bitmap lock (see drbd_bitmap_io).
* For lazy writeout, we don't care for ongoing changes to the bitmap,
* as we submit copies of pages anyways.
*/
ctx = kmalloc(sizeof(struct drbd_bm_aio_ctx), GFP_NOIO);
if (!ctx)
return -ENOMEM;
*ctx = (struct drbd_bm_aio_ctx) {
.device = device,
.start_jif = jiffies,
.in_flight = ATOMIC_INIT(1),
.done = 0,
.flags = flags,
.error = 0,
.kref = KREF_INIT(2),
};
if (!get_ldev_if_state(device, D_ATTACHING)) { /* put is in drbd_bm_aio_ctx_destroy() */
drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in bm_rw()\n");
kfree(ctx);
return -ENODEV;
}
/* Here D_ATTACHING is sufficient since drbd_bm_read() is called only from
drbd_adm_attach(), after device->ldev was assigned. */
if (0 == (ctx->flags & ~BM_AIO_READ))
WARN_ON(!(BM_LOCKED_MASK & b->bm_flags));
spin_lock_irq(&device->resource->req_lock);
list_add_tail(&ctx->list, &device->pending_bitmap_io);
spin_unlock_irq(&device->resource->req_lock);
num_pages = b->bm_number_of_pages;
now = jiffies;
/* let the layers below us try to merge these bios... */
if (flags & BM_AIO_READ) {
for (i = 0; i < num_pages; i++) {
atomic_inc(&ctx->in_flight);
bm_page_io_async(ctx, i);
++count;
cond_resched();
}
} else if (flags & BM_AIO_WRITE_HINTED) {
/* ASSERT: BM_AIO_WRITE_ALL_PAGES is not set. */
unsigned int hint;
for (hint = 0; hint < b->n_bitmap_hints; hint++) {
i = b->al_bitmap_hints[hint];
if (i >= num_pages) /* == -1U: no hint here. */
continue;
/* Several AL-extents may point to the same page. */
if (!test_and_clear_bit(BM_PAGE_HINT_WRITEOUT,
&page_private(b->bm_pages[i])))
continue;
/* Has it even changed? */
if (bm_test_page_unchanged(b->bm_pages[i]))
continue;
atomic_inc(&ctx->in_flight);
bm_page_io_async(ctx, i);
++count;
}
} else {
for (i = 0; i < num_pages; i++) {
/* ignore completely unchanged pages */
if (lazy_writeout_upper_idx && i == lazy_writeout_upper_idx)
break;
if (!(flags & BM_AIO_WRITE_ALL_PAGES) &&
bm_test_page_unchanged(b->bm_pages[i])) {
dynamic_drbd_dbg(device, "skipped bm write for idx %u\n", i);
continue;
}
/* during lazy writeout,
* ignore those pages not marked for lazy writeout. */
if (lazy_writeout_upper_idx &&
!bm_test_page_lazy_writeout(b->bm_pages[i])) {
dynamic_drbd_dbg(device, "skipped bm lazy write for idx %u\n", i);
continue;
}
atomic_inc(&ctx->in_flight);
bm_page_io_async(ctx, i);
++count;
cond_resched();
}
}
/*
* We initialize ctx->in_flight to one to make sure drbd_bm_endio
* will not set ctx->done early, and decrement / test it here. If there
* are still some bios in flight, we need to wait for them here.
* If all IO is done already (or nothing had been submitted), there is
* no need to wait. Still, we need to put the kref associated with the
* "in_flight reached zero, all done" event.
*/
if (!atomic_dec_and_test(&ctx->in_flight))
wait_until_done_or_force_detached(device, device->ldev, &ctx->done);
else
kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy);
/* summary for global bitmap IO */
if (flags == 0) {
unsigned int ms = jiffies_to_msecs(jiffies - now);
if (ms > 5) {
drbd_info(device, "bitmap %s of %u pages took %u ms\n",
(flags & BM_AIO_READ) ? "READ" : "WRITE",
count, ms);
}
}
if (ctx->error) {
drbd_alert(device, "we had at least one MD IO ERROR during bitmap IO\n");
drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
err = -EIO; /* ctx->error ? */
}
if (atomic_read(&ctx->in_flight))
err = -EIO; /* Disk timeout/force-detach during IO... */
now = jiffies;
if (flags & BM_AIO_READ) {
b->bm_set = bm_count_bits(b);
drbd_info(device, "recounting of set bits took additional %lu jiffies\n",
jiffies - now);
}
now = b->bm_set;
if ((flags & ~BM_AIO_READ) == 0)
drbd_info(device, "%s (%lu bits) marked out-of-sync by on disk bit-map.\n",
ppsize(ppb, now << (BM_BLOCK_SHIFT-10)), now);
kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy);
return err;
}