in dm-integrity.c [2103:2322]
static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)
{
struct dm_integrity_c *ic = dio->ic;
struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
unsigned journal_section, journal_entry;
unsigned journal_read_pos;
struct completion read_comp;
bool discard_retried = false;
bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;
if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')
need_sync_io = true;
if (need_sync_io && from_map) {
INIT_WORK(&dio->work, integrity_bio_wait);
queue_work(ic->offload_wq, &dio->work);
return;
}
lock_retry:
spin_lock_irq(&ic->endio_wait.lock);
retry:
if (unlikely(dm_integrity_failed(ic))) {
spin_unlock_irq(&ic->endio_wait.lock);
do_endio(ic, bio);
return;
}
dio->range.n_sectors = bio_sectors(bio);
journal_read_pos = NOT_FOUND;
if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {
if (dio->op == REQ_OP_WRITE) {
unsigned next_entry, i, pos;
unsigned ws, we, range_sectors;
dio->range.n_sectors = min(dio->range.n_sectors,
(sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block);
if (unlikely(!dio->range.n_sectors)) {
if (from_map)
goto offload_to_thread;
sleep_on_endio_wait(ic);
goto retry;
}
range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;
ic->free_sectors -= range_sectors;
journal_section = ic->free_section;
journal_entry = ic->free_section_entry;
next_entry = ic->free_section_entry + range_sectors;
ic->free_section_entry = next_entry % ic->journal_section_entries;
ic->free_section += next_entry / ic->journal_section_entries;
ic->n_uncommitted_sections += next_entry / ic->journal_section_entries;
wraparound_section(ic, &ic->free_section);
pos = journal_section * ic->journal_section_entries + journal_entry;
ws = journal_section;
we = journal_entry;
i = 0;
do {
struct journal_entry *je;
add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);
pos++;
if (unlikely(pos >= ic->journal_entries))
pos = 0;
je = access_journal_entry(ic, ws, we);
BUG_ON(!journal_entry_is_unused(je));
journal_entry_set_inprogress(je);
we++;
if (unlikely(we == ic->journal_section_entries)) {
we = 0;
ws++;
wraparound_section(ic, &ws);
}
} while ((i += ic->sectors_per_block) < dio->range.n_sectors);
spin_unlock_irq(&ic->endio_wait.lock);
goto journal_read_write;
} else {
sector_t next_sector;
journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
if (likely(journal_read_pos == NOT_FOUND)) {
if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))
dio->range.n_sectors = next_sector - dio->range.logical_sector;
} else {
unsigned i;
unsigned jp = journal_read_pos + 1;
for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {
if (!test_journal_node(ic, jp, dio->range.logical_sector + i))
break;
}
dio->range.n_sectors = i;
}
}
}
if (unlikely(!add_new_range(ic, &dio->range, true))) {
/*
* We must not sleep in the request routine because it could
* stall bios on current->bio_list.
* So, we offload the bio to a workqueue if we have to sleep.
*/
if (from_map) {
offload_to_thread:
spin_unlock_irq(&ic->endio_wait.lock);
INIT_WORK(&dio->work, integrity_bio_wait);
queue_work(ic->wait_wq, &dio->work);
return;
}
if (journal_read_pos != NOT_FOUND)
dio->range.n_sectors = ic->sectors_per_block;
wait_and_add_new_range(ic, &dio->range);
/*
* wait_and_add_new_range drops the spinlock, so the journal
* may have been changed arbitrarily. We need to recheck.
* To simplify the code, we restrict I/O size to just one block.
*/
if (journal_read_pos != NOT_FOUND) {
sector_t next_sector;
unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
if (unlikely(new_pos != journal_read_pos)) {
remove_range_unlocked(ic, &dio->range);
goto retry;
}
}
}
if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {
sector_t next_sector;
unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
if (unlikely(new_pos != NOT_FOUND) ||
unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {
remove_range_unlocked(ic, &dio->range);
spin_unlock_irq(&ic->endio_wait.lock);
queue_work(ic->commit_wq, &ic->commit_work);
flush_workqueue(ic->commit_wq);
queue_work(ic->writer_wq, &ic->writer_work);
flush_workqueue(ic->writer_wq);
discard_retried = true;
goto lock_retry;
}
}
spin_unlock_irq(&ic->endio_wait.lock);
if (unlikely(journal_read_pos != NOT_FOUND)) {
journal_section = journal_read_pos / ic->journal_section_entries;
journal_entry = journal_read_pos % ic->journal_section_entries;
goto journal_read_write;
}
if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {
if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
struct bitmap_block_status *bbs;
bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);
spin_lock(&bbs->bio_queue_lock);
bio_list_add(&bbs->bio_queue, bio);
spin_unlock(&bbs->bio_queue_lock);
queue_work(ic->writer_wq, &bbs->work);
return;
}
}
dio->in_flight = (atomic_t)ATOMIC_INIT(2);
if (need_sync_io) {
init_completion(&read_comp);
dio->completion = &read_comp;
} else
dio->completion = NULL;
dm_bio_record(&dio->bio_details, bio);
bio_set_dev(bio, ic->dev->bdev);
bio->bi_integrity = NULL;
bio->bi_opf &= ~REQ_INTEGRITY;
bio->bi_end_io = integrity_end_io;
bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {
integrity_metadata(&dio->work);
dm_integrity_flush_buffers(ic, false);
dio->in_flight = (atomic_t)ATOMIC_INIT(1);
dio->completion = NULL;
submit_bio_noacct(bio);
return;
}
submit_bio_noacct(bio);
if (need_sync_io) {
wait_for_completion_io(&read_comp);
if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))
goto skip_check;
if (ic->mode == 'B') {
if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,
dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))
goto skip_check;
}
if (likely(!bio->bi_status))
integrity_metadata(&dio->work);
else
skip_check:
dec_in_flight(dio);
} else {
INIT_WORK(&dio->work, integrity_metadata);
queue_work(ic->metadata_wq, &dio->work);
}
return;
journal_read_write:
if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))
goto lock_retry;
do_endio_flush(ic, dio);
}