gmane.linux.kernel.device-mapper.devel

Re: Calltrace in dm-snapshot in 2.6.27 kernel

Mikulas Patocka — 2008-12-02T02:10:19

Hi I don't know really what is causing it. Just try this patch. It removes a lot of dirty logic from dm-snapshots. (you might need to hand-edit the patch a little bit depending on other patches that you have applied). Mikulas dm-snapshot-rework-origin-write.patch: Rework writing to snapshot origin. The previous code selected one exception as "primary_pe", linked all other exceptions on it and used reference counting to wait until all exceptions are reallocated. This didn't work with exceptions with different chunk sizes: https://bugzilla.redhat.com/show_bug.cgi?id=182659 I removed all the complexity with exceptions linking and reference counting. Currently, bio is linked on one exception and when that exception is reallocated, the bio is retried to possibly wait for other exceptions. Signed-off-by: Mikulas Patocka redhat.com> --- drivers/md/dm-snap.c | 174 +++++++++++++++++---------------------------------- 1 file changed, 61 insertions(+), 113 deletions(-) Index: linux-2.6.28-rc5-devel/drivers/md/dm-snap.c =================================================================== --- linux-2.6.28-rc5-devel.orig/drivers/md/dm-snap.c2008-11-25 16:10:37.000000000 +0100 +++ linux-2.6.28-rc5-devel/drivers/md/dm-snap.c2008-11-25 16:10:42.000000000 +0100 < at >< at > -56,28 +56,6 < at >< at > struct dm_snap_pending_exception { struct bio_list origin_bios; struct bio_list snapshot_bios; -/* - * Short-term queue of pending exceptions prior to submission. - */ -struct list_head list; - -/* - * The primary pending_exception is the one that holds - * the ref_count and the list of origin_bios for a - * group of pending_exceptions. It is always last to get freed. - * These fields get set up when writing to the origin. - */ -struct dm_snap_pending_exception *primary_pe; - -/* - * Number of pending_exceptions processing this chunk. - * When this drops to zero we must complete the origin bios. - * If incrementing or decrementing this, hold pe->snap->lock for - * the sibling concerned and not pe->primary_pe->snap->lock unless - * they are the same. - */ -atomic_t ref_count; - /* Pointer back to snapshot context */ struct dm_snapshot *snap; < at >< at > -758,6 +736,28 < at >< at > static void flush_bios(struct bio *bio) } } +static int do_origin(struct dm_dev *origin, struct bio *bio); + +/* + * Flush a list of buffers. + */ +static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio) +{ +struct bio *n; +int r; + +while (bio) { +n = bio->bi_next; +bio->bi_next = NULL; +r = do_origin(s->origin, bio); +if (r == DM_MAPIO_REMAPPED) +generic_make_request(bio); +else +BUG_ON(r != DM_MAPIO_SUBMITTED); +bio = n; +} +} + /* * Error a list of buffers. */ < at >< at > -791,39 +791,6 < at >< at > static void __invalidate_snapshot(struct dm_table_event(s->ti->table); } -static void get_pending_exception(struct dm_snap_pending_exception *pe) -{ -atomic_inc(&pe->ref_count); -} - -static struct bio *put_pending_exception(struct dm_snap_pending_exception *pe) -{ -struct dm_snap_pending_exception *primary_pe; -struct bio *origin_bios = NULL; - -primary_pe = pe->primary_pe; - -/* - * If this pe is involved in a write to the origin and - * it is the last sibling to complete then release - * the bios for the original write to the origin. - */ -if (primary_pe && - atomic_dec_and_test(&primary_pe->ref_count)) { -origin_bios = bio_list_get(&primary_pe->origin_bios); -free_pending_exception(primary_pe); -} - -/* - * Free the pe if it's not linked to an origin write or if - * it's not itself a primary pe. - */ -if (!primary_pe || primary_pe != pe) -free_pending_exception(pe); - -return origin_bios; -} - static void pending_complete(struct dm_snap_pending_exception *pe, int success) { struct dm_snap_exception *e; < at >< at > -872,7 +839,8 < at >< at > static void pending_complete(struct dm_s out: remove_exception(&pe->e); snapshot_bios = bio_list_get(&pe->snapshot_bios); -origin_bios = put_pending_exception(pe); +origin_bios = bio_list_get(&pe->origin_bios); +free_pending_exception(pe); up_write(&s->lock); < at >< at > -882,7 +850,7 < at >< at > static void pending_complete(struct dm_s else flush_bios(snapshot_bios); -flush_bios(origin_bios); +retry_origin_bios(s, origin_bios); } static void commit_callback(void *context, int success) < at >< at > -944,11 +912,11 < at >< at > static void start_copy(struct dm_snap_pe * this. */ static struct dm_snap_pending_exception * -__find_pending_exception(struct dm_snapshot *s, struct bio *bio) +__find_pending_exception(struct dm_snapshot *s, sector_t sector) { struct dm_snap_exception *e; struct dm_snap_pending_exception *pe; -chunk_t chunk = sector_to_chunk(s, bio->bi_sector); +chunk_t chunk = sector_to_chunk(s, sector); /* * Is there a pending exception for this already ? < at >< at > -983,8 +951,6 < at >< at > __find_pending_exception(struct dm_snaps pe->e.old_chunk = chunk; bio_list_init(&pe->origin_bios); bio_list_init(&pe->snapshot_bios); -pe->primary_pe = NULL; -atomic_set(&pe->ref_count, 0); pe->started = 0; if (s->store.prepare_exception(&s->store, &pe->e)) { < at >< at > -992,7 +958,6 < at >< at > __find_pending_exception(struct dm_snaps return NULL; } -get_pending_exception(pe); insert_exception(&s->pending, &pe->e); out: < at >< at > -1046,7 +1011,7 < at >< at > static int snapshot_map(struct dm_target * writeable. */ if (bio_rw(bio) == WRITE) { -pe = __find_pending_exception(s, bio); +pe = __find_pending_exception(s, bio->bi_sector); if (!pe) { __invalidate_snapshot(s, -ENOMEM); r = -EIO; < at >< at > -1140,14 +1105,20 < at >< at > static int snapshot_status(struct dm_tar /*----------------------------------------------------------------- * Origin methods *---------------------------------------------------------------*/ -static int __origin_write(struct list_head *snapshots, struct bio *bio) + +/* + * Returns: + *DM_MAPIO_REMAPPED: bio may be submitted to origin device + *DM_MAPIO_SUBMITTED: bio was queued on queue on one of exceptions + */ + +static int __origin_write(struct list_head *snapshots, sector_t sector, struct bio *bio) { -int r = DM_MAPIO_REMAPPED, first = 0; +int r = DM_MAPIO_REMAPPED; struct dm_snapshot *snap; struct dm_snap_exception *e; -struct dm_snap_pending_exception *pe, *next_pe, *primary_pe = NULL; +struct dm_snap_pending_exception *pe, *pe_to_start = NULL; chunk_t chunk; -LIST_HEAD(pe_queue); /* Do all the snapshots on this origin */ list_for_each_entry (snap, snapshots, list) { < at >< at > -1159,86 +1130,63 < at >< at > static int __origin_write(struct list_he goto next_snapshot; /* Nothing to do if writing beyond end of snapshot */ -if (bio->bi_sector >= dm_table_get_size(snap->ti->table)) +if (sector >= dm_table_get_size(snap->ti->table)) goto next_snapshot; /* * Remember, different snapshots can have * different chunk sizes. */ -chunk = sector_to_chunk(snap, bio->bi_sector); +chunk = sector_to_chunk(snap, sector); /* * Check exception table to see if block * is already remapped in this snapshot * and trigger an exception if not. - * - * ref_count is initialised to 1 so pending_complete() - * won't destroy the primary_pe while we're inside this loop. */ e = lookup_exception(&snap->complete, chunk); if (e) goto next_snapshot; -pe = __find_pending_exception(snap, bio); +pe = __find_pending_exception(snap, sector); if (!pe) { __invalidate_snapshot(snap, -ENOMEM); goto next_snapshot; } -if (!primary_pe) { -/* - * Either every pe here has same - * primary_pe or none has one yet. - */ -if (pe->primary_pe) -primary_pe = pe->primary_pe; -else { -primary_pe = pe; -first = 1; -} - -bio_list_add(&primary_pe->origin_bios, bio); - -r = DM_MAPIO_SUBMITTED; -} +r = DM_MAPIO_SUBMITTED; -if (!pe->primary_pe) { -pe->primary_pe = primary_pe; -get_pending_exception(primary_pe); +if (bio) { +bio_list_add(&pe->origin_bios, bio); +bio = NULL; + +if (!pe->started) { +pe->started = 1; +pe_to_start = pe; +} } if (!pe->started) { pe->started = 1; -list_add_tail(&pe->list, &pe_queue); +start_copy(pe); } next_snapshot: up_write(&snap->lock); } -if (!primary_pe) -return r; - /* - * If this is the first time we're processing this chunk and - * ref_count is now 1 it means all the pending exceptions - * got completed while we were in the loop above, so it falls to - * us here to remove the primary_pe and submit any origin_bios. + * pe_to_start is a small performance improvement: + * To avoid calling __origin_write N times for N snapshots, we start + * the snapshot where we queued the bio as the last one. + * + * If we start it as the last one, it finishes most likely as the last + * one and exceptions in other snapshots will be already finished when + * the bio will be retried. */ -if (first && atomic_dec_and_test(&primary_pe->ref_count)) { -flush_bios(bio_list_get(&primary_pe->origin_bios)); -free_pending_exception(primary_pe); -/* If we got here, pe_queue is necessarily empty. */ -return r; -} - -/* - * Now that we have a complete pe list we can start the copying. - */ -list_for_each_entry_safe(pe, next_pe, &pe_queue, list) -start_copy(pe); +if (pe_to_start) +start_copy(pe_to_start); return r; } < at >< at > -1254,7 +1202,7 < at >< at > static int do_origin(struct dm_dev *orig down_read(&_origins_lock); o = __lookup_origin(origin->bdev); if (o) -r = __origin_write(&o->snapshots, bio); +r = __origin_write(&o->snapshots, bio->bi_sector, bio); up_read(&_origins_lock); return r; On Fri, 28 Nov 2008, aluno3< at >poczta.onet.pl wrote:

Re: [RFC][PATCH 0/4] dm-log: support multi-log devices

Phillip Susi — 2008-12-01T16:10:01

Once the IO request has been completed, the data needs to be stable on the disk. This means that either you have to wait until the data has been written to all underlying mirror devices before completing the request ( slow ) or you have to have some way of knowing which disk(s) got written to, and which ones need updated after a crash. Are you saying you take the former path? Does the entire log-data-log update cycle complete before dm completes the higher level IO request? That would maintain data integrity, but at significant cost to performance. For performance sake, don't you want to allow write requests to be completed before the log is necessarily marked as clean again? That way multiple writes to the same data zone do not require multiple log dirty/clean updates. Also for performance reasons, don't you want to allow the data to be written to only one mirror before completing the request? Then go back and do lazy synchronization?

Re: [RFC][PATCH 0/4] dm-log: support multi-log devices

Michał Mirosław — 2008-12-01T11:09:37

2008/12/1 Takahiro Yasui redhat.com>: [...] What happens if some log devices contain "dirty" and not all mirrors were written yet before a crash? How do you know which mirror has the most recent data? Are the writes to mirrors ordered somehow? Best Regards, Michal Miroslaw

Re: [RFC][PATCH 2/4] dm-log: unify rw_header toread/write_header

Takahiro Yasui — 2008-12-01T07:24:01

Oh, I'm sorry not to mention it on the patch description, but your explanation is exactly what I need to do. Thanks, Alasdair. I have removed rw_header function in my patch since the last two patches don't use it, but your improvement is fine as a separate patch. Thanks, --- Takahiro Yasui Hitachi Computer Products (America) Inc.

Re: Re: [RFC][PATCH 0/4] dm-log: support multi-log devices

Takahiro Yasui — 2008-12-01T07:00:54

log disks are updated in parallel and we do not know which disk has the latest and correct data if the system crashes during write operations on log devices. But there is no problem about it. There are two cases we need to think about. 1) Some log devices contain "clean", but mirror devices are not synchronized This case is problematic, but never happens, because data is written on mirror devices after marking log devices "dirty", and make it "clean" after write I/Os on mirror devices completed and mirrors get synchronized. 2) Some log devices contain "dirty", but mirror devices are synchronized This case may happen but is not problematic. Just data replication of the region among mirror devices will be done when the mirror is resumed. This case would also happen on the system with the current single log if the system crashes after marking a log device "dirty" and before marking it back to "clean". Thanks, --- Takahiro Yasui Hitachi Computer Products (America) Inc.

Re: Re: next-20081125: crypto hard disk gets unaccessable

Herbert Xu — 2008-12-01T04:57:49

Yes that's definitely synchronous. Cheers,

Re: Re: next-20081125: crypto hard disk gets unaccessable

Milan Broz — 2008-12-01T00:16:06

Did this work before? If so, please can you provide version where it works? It is suspend to encrypted swap? There should not be any recent change in dm-crypt related to this path, so I expect it is some bug related to suspend/resume in combination with running crypto - like corruption of some memory caused during suspend... I assume that crypto run here synchronously (not through asynchronous completion callback). Herbert - is my guess ok? (The asynchronous handling in dm-crypt was the only part changed recently.) (And I think that except crypto hw nobody uses asynchronous mode yet by default.) Milan -- mbroz< at >redhat.com

Re: next-20081125: crypto hard disk gets unaccessable

Herbert Xu — 2008-11-30T04:32:15

8b 04 0e mov (%rsi,%rcx,1),%eax So we were fed a bogus pointer by the caller. Nothing has changed in the crypto ciphers lately. Cheers,

Re: next-20081125: crypto hard disk gets unaccessable

Andrew Morton — 2008-11-30T03:29:39

(cc's added) On Fri, 28 Nov 2008 07:33:55 +0100 Nico -telmich- Schottelius schottelius.org> wrote: [lots of snippage] Either a bug in dm-crypt or in crypto core, I'd guess. I assume that 2.6.28-rc6 is OK?

Re: [RFC][PATCH 2/4] dm-log: unify rw_header toread/write_header

Alasdair G Kergon — 2008-11-28T23:37:47

Indeed. And it also removes header_to_disk() from the disk_flush() code path but doesn't mention changing that:-) I assume that's also OK because there'll always have been a resume before the flush and the data can only change during a resume? I reckon that little function improves readability/maintainability and I've left it in. http://www.kernel.org/pub/linux/kernel/people/agk/patches/2.6/editing/ dm-log-avoid-reinitialising-io_req-on-every-operation.patch Alasdair

Re: Re: [RFC][PATCH 0/4] dm-log: support multi-log devices

Phillip Susi — 2008-11-28T20:06:25

Right... and when recording a log on every disk in the mirror, each copy of the log may not contain exactly the same information at any given time. If you write to one disk in the mirror first, then you need to mark the region as dirty on that disk first, so that if the system crashes before you can copy the data to the other mirror, you can see that the first disk is more up to date than the second disk. In other words, knowing that a region is or is not synchronized across each disk is not enough; if they are out of sync you need to figure out which disk has the most current information so it can be replicated to the others, don't you? Or do you just always write to the first disk first, and assume it has the most recent data if the region was marked as dirty in ANY of the logs?

Re:Announce: unlimited number of shared snapshots

Mikulas Patocka — 2008-11-28T19:54:03

Oh, sorry, there was a bug with big devices. I uploaded new patches at the same location, try them. (there's another fix for bug when sector size is smaller than chunk size in these new patches) BTW. for good performance, make sure that the size of your origin partition is aligned to the chunk size --- otherwise, there's a serious inefficiency in the kernel; if you use it over a partition with odd number of sectors, the kernel will split all IOs to 512 bytes and it'll be very slow. Mikulas

Re: Calltrace in dm-snapshot in 2.6.27 kernel

2008-11-28T07:28:16

Hi More info about problem: In my previous mail I wrote that I brought to call trace after a few minutes but without use "dm snapshot: fix primary_pe race"and "wait for chunks in destructor" of course and that one call trace ("after a few minutes") involve problem with "wait for chunks in destructor". But,yesterday I got next call trace: Pid: 25597, comm: kcopyd Not tainted (2.6.27.7 #47) EIP: 0060:[] EFLAGS: 00010282 CPU: 1 EIP is at remove_exception+0x5/0x20 EAX: c7b3d348 EBX: c7b3d348 ECX: 00200200 EDX: 00100100 ESI: c799d770 EDI: ea6a99c0 EBP: 00000000 ESP: f26b5ec0 DS: 007b ES: 007b FS: 00d8 GS: 0000 SS: 0068 Process kcopyd (pid: 25597, ti=f26b4000 task=f09e11f0 task.ti=f26b4000) Stack: c044e04f 0000000c 00000000 f1980c40 00000000 c044f313 001396a7 00000000 000013ba 00000000 c7b3dd88 ea6a99c0 00000000 c044e100 c7b3dd88 f30f7aa8 00000000 c0449234 00000000 f30f7b94 00000400 00000000 00000000 ea53fcb8 Call Trace: [] pending_complete+0x9f/0x110 [] persistent_commit+0xe3/0x110 [] copy_callback+0x30/0x40 [] segment_complete+0x154/0x1d0 [] run_complete_job+0x45/0x80 [] segment_complete+0x0/0x1d0 [] run_complete_job+0x0/0x80 [] process_jobs+0x14/0x70 [] do_work+0x0/0x40 [] do_work+0x16/0x40 [] run_workqueue+0x4d/0xf0 [] worker_thread+0x7d/0xc0 [] autoremove_wake_function+0x0/0x30 [] __sched_text_start+0x1e3/0x4a0 [] autoremove_wake_function+0x0/0x30 [] complete+0x2b/0x40 [] worker_thread+0x0/0xc0 [] kthread+0x44/0x70 [] kthread+0x0/0x70 [] kernel_thread_helper+0x7/0x10 ======================= Code: 4b 0c e8 cf ff ff ff 8b 56 08 8d 04 c2 8b 10 89 13 89 18 89 5a 04 89 43 04 5b 5e c3 8d 76 00 8d bc 27 00 00 00 00 8b 48 04 8b 10 <89> 11 89 4a 04 c7 00 00 01 10 00 c7 40 04 00 02 20 00 c3 90 8d EIP: [] remove_exception+0x5/0x20 SS:ESP 0068:f26b5ec0 ---[ end trace 8a6182ef9a00114f ]--- root< at >53434231:~# uname -a Linux 53434231 2.6.27.7 #47 SMP Tue Nov 25 08:57:37 CET 2008 i686 GNU/Linux root< at >53434231:~# addr2line -e ./vmlinux c044d495 include/linux/list.h:93 git log for my drivers/md/ show in order: dm snapshot: wait for chunks in destructor dm snapshot: fix register_snapshot deadlock dm snapshot: drop unused last_percent dm raid1: flush workqueue before destruction md: fix bug in raid10 recovery. md: linear: Fix a division by zero bug for very small arrays. dm snapshot: fix primary_pe race dm kcopyd: avoid queue shuffle md: Fix rdev_size_store with size == 0 ... aluno3< at >poczta.onet.pl wrote:

Re: [PATCH 0/9] dm snapshot: shared exception store (v4)

Mikulas Patocka — 2008-11-28T06:22:50

Hi Look at this http://people.redhat.com/mpatocka/patches/kernel/new-snapshots/ It is standalone implementation, without any dependency on old snapshots. Would you like to create patches against this instead of the original snapshots? Mikulas On Thu, 27 Nov 2008, FUJITA Tomonori wrote:

RE: Re: multipathing pending issues with rhel

Christophe Varoqui — 2008-11-27T22:16:25

I recently tried an upstream kernel with the device handlers infrastructure, and it seems the kernel rightfully drops the io on reservation conflicts ... with due message in the logs, and returning a success to the devmapper. No queueing, no retry, no problem. I'll have to do some more testing to convince myself, but in the mean time : 1/ Do you confirm my observations ? 2/ Does the Clariion snapshot still need your suggested fix in this new context ? If so, would you care to submit a patch ? Thanks for the comment, cvaroqui

path_checker within multipath {}

Matthew Kent — 2008-11-27T19:47:54

Am I missing something obvious here? /usr/share/doc/device-mapper-multipath-0.4.7/multipath.conf.annotated says this should work, and I can't find a RHEL 5 issue for it: My config says: multipaths { multipath { wwid "22263000155ada9e9" path_grouping_policy failover path_checker tur } multipath { wwid "36090a018108277e8ed92e4e9cee1733e" path_grouping_policy multibus path_checker readsector0 } } but multipathd -k"show conf" says: multipaths { multipath { wwid 22263000155ada9e9 path_grouping_policy failover } multipath { wwid 36090a018108277e8ed92e4e9cee1733e path_grouping_policy multibus } and multipath -v3 shows them all sdb: path checker = readsector0 (config file default) while the grouping policy seems to work, pgpolicy = multibus (LUN setting) I'm sure I can use 'tur' on both multipaths via the defaults {} section and be fine, but from the docs (even checked the RHEL 5.3 beta SRPM) this should work.

Re: Re: [RFC][PATCH 0/4] dm-log: support multi-log devices

Takahiro Yasui — 2008-11-27T07:50:39

bitmap data on a log disk indicates if each region are "clean" or "dirty" among mirror disks. If a bit related to a region is "1", it means that the region is "clean" and data in the region is synchronized in all mirror disks and they store the same data in the region. On the other hand, if a bit related to a region is "0", the region is "dirty" and the region is out of synchronization. It means that each mirror disk might contain different data in the region. Therefore, a log disk contains a state of each region, but does not correspond to a specific mirror device. This patch set introduces redundant log disk on dm-mirror. Thanks, --- Takahiro Yasui Hitachi Computer Products (America) Inc.

Announce: unlimited number of shared snapshots

Mikulas Patocka — 2008-11-27T05:41:36

Hi I made first release of my snapshot storage that can hold unlimited number of snapshots and share data between them: http://people.redhat.com/mpatocka/patches/kernel/new-snapshots/ How-to-use is at the beginning of dm-multisnapshot.patch If you see some errors, report them. This storage uses btree with the key (chunk number, snapshot range). It uses log-structured storage, so that it is safe w.r.t. crash. TODO: - reclaim allocated space, allow deleting snapshots - writable snapshots Mikulas

Re: [PATCHES] new solution for dm_any_congested crash

jvrao — 2008-11-26T21:41:32

Hi, I was the one who initially stumbled onto this problem, and when I realized that it is the ABBA deadlock, I approached Chandra and Mike. Chandra came up with the initial fix, and it worked fine. Later Chandra pointed me to this patch, and when I tried it on.. I ran into system hang. Please note that, I am running it on -rt kernel based on 2.6.24. I could not apply the patch directly, so I ported it onto my kernel. I am attaching the ported version(new_dm_patch)..I already ran this ported patch by Chandra. I ran IO stress test with this patch while one of the paths is constantly bounced . Bounced the same path all the time. (20min between the bounces) System hung few hours into the test..and I forced the dump. I am still analyzing the dump. If you want to have the dump, please let me know where I can upload it to. Core is around 8G Here are few things from the dump that might be interesting. crash> ps |grep udev | wc -l 425 << <<<425 udevd threads at the time of hang. crash> ps | wc -l 782 crash> foreach bt| grep rt_mutex_slowlock | wc -l 416 < <<<<416 of the the total 782 threads are waiting for a lock. crash> crash> struct rt_mutex ffff81024ec6cca0 struct rt_mutex { wait_lock = { raw_lock = { slock = 49858 }, break_lock = 0 }, wait_list = { prio_list = { next = 0xffff8101007f5ae0, prev = 0xffff8101007f5ae0 }, node_list = { next = 0xffff8101007f5af0, prev = 0xffff81007cb51af0 } }, owner = 0xffff8102400c2b22 } Following task is holding the lock that many other udevs are waiting for. PID: 21896 TASK: ffff8102400c2b20 CPU: 6 COMMAND: "udevd" << Holding #0 [ffff810100667848] schedule at ffffffff8128531c #1 [ffff810100667900] io_schedule at ffffffff81285859 #2 [ffff810100667920] sync_buffer at ffffffff810d1fc1 #3 [ffff810100667930] __wait_on_bit at ffffffff81285ad1 #4 [ffff810100667970] out_of_line_wait_on_bit at ffffffff81285b71 #5 [ffff8101006679e0] __wait_on_buffer at ffffffff810d1f41 #6 [ffff8101006679f0] ext3_find_entry at ffffffff8803c1d2 #7 [ffff810100667b60] ext3_lookup at ffffffff8803dbae #8 [ffff810100667ba0] do_lookup at ffffffff810b78cf #9 [ffff810100667bf0] __link_path_walk at ffffffff810b94ef #10 [ffff810100667c90] link_path_walk at ffffffff810b9f99 #11 [ffff810100667d60] path_walk at ffffffff810ba04b #12 [ffff810100667d70] do_path_lookup at ffffffff810ba352 #13 [ffff810100667dc0] __path_lookup_intent_open at ffffffff810bae88 #14 [ffff810100667e10] path_lookup_open at ffffffff810baf38 #15 [ffff810100667e20] open_exec at ffffffff810b41e3 #16 [ffff810100667ed0] do_execve at ffffffff810b53a2 #17 [ffff810100667f20] sys_execve at ffffffff8100ac30 #18 [ffff810100667f50] stub_execve at ffffffff8100c5c7 PID: 21946 TASK: ffff81007f090b20 CPU: 4 COMMAND: "udevd" << One of the udevds waiting for the lock. #0 [ffff81007f0e59f8] schedule at ffffffff8128531c #1 [ffff81007f0e5ab0] rt_mutex_slowlock at ffffffff81286a95 #2 [ffff81007f0e5b80] rt_mutex_lock at ffffffff81285f84 #3 [ffff81007f0e5b90] _mutex_lock at ffffffff812873f9 #4 [ffff81007f0e5ba0] do_lookup at ffffffff810b788b #5 [ffff81007f0e5bf0] __link_path_walk at ffffffff810b94ef #6 [ffff81007f0e5c90] link_path_walk at ffffffff810b9f99 #7 [ffff81007f0e5d60] path_walk at ffffffff810ba04b #8 [ffff81007f0e5d70] do_path_lookup at ffffffff810ba352 #9 [ffff81007f0e5dc0] __path_lookup_intent_open at ffffffff810bae88 #10 [ffff81007f0e5e10] path_lookup_open at ffffffff810baf38 #11 [ffff81007f0e5e20] open_exec at ffffffff810b41e3 #12 [ffff81007f0e5ed0] do_execve at ffffffff810b53a2 #13 [ffff81007f0e5f20] sys_execve at ffffffff8100ac30 #14 [ffff81007f0e5f50] stub_execve at ffffffff8100c5c7 Thanks, Venkateswararao Jujjuri (JV) Realtime Team, LTC, Beaverton, OR 97006 Index: linux-2.6.24.7-ibmrt2.16-view/drivers/md/dm.c =================================================================== --- linux-2.6.24.7-ibmrt2.16-view.orig/drivers/md/dm.c +++ linux-2.6.24.7-ibmrt2.16-view/drivers/md/dm.c < at >< at > -876,20 +876,16 < at >< at > static void dm_unplug_all(struct request static int dm_any_congested(void *congested_data, int bdi_bits) { -int r = bdi_bits; +int r; struct mapped_device *md = (struct mapped_device *) congested_data; -struct dm_table *map; +struct dm_table *map = dm_get_table(md); -atomic_inc(&md->pending); -if (!test_bit(DMF_BLOCK_IO, &md->flags)) { -map = dm_get_table(md); -if (map) { -r = dm_table_any_congested(map, bdi_bits); -dm_table_put(map); -} -} -if (!atomic_dec_return(&md->pending)) -wake_up(&md->wait); +if (!map || test_bit(DMF_BLOCK_IO, &md->flags)) +r = bdi_bits; +else +r = dm_table_any_congested(map, bdi_bits); + +dm_table_put(map); return r; } < at >< at > -1143,10 +1139,11 < at >< at > static int __bind(struct mapped_device * if (md->suspended_bdev) __set_size(md, size); -if (size == 0) +if (size == 0) { +dm_table_destroy(t); return 0; +} -dm_table_get(t); dm_table_event_callback(t, event_callback, md); write_lock(&md->map_lock); < at >< at > -1168,7 +1165,7 < at >< at > static void __unbind(struct mapped_devic write_lock(&md->map_lock); md->map = NULL; write_unlock(&md->map_lock); -dm_table_put(map); +dm_table_destroy(map); } /* < at >< at > -1256,8 +1253,8 < at >< at > void dm_put(struct mapped_device *md) dm_table_presuspend_targets(map); dm_table_postsuspend_targets(map); } -__unbind(md); dm_table_put(map); +__unbind(md); free_dev(md); } } Index: linux-2.6.24.7-ibmrt2.16-view/drivers/md/dm-ioctl.c =================================================================== --- linux-2.6.24.7-ibmrt2.16-view.orig/drivers/md/dm-ioctl.c +++ linux-2.6.24.7-ibmrt2.16-view/drivers/md/dm-ioctl.c < at >< at > -232,7 +232,7 < at >< at > static void __hash_remove(struct hash_ce } if (hc->new_map) -dm_table_put(hc->new_map); +dm_table_destroy(hc->new_map); dm_put(hc->md); free_cell(hc); } < at >< at > -826,8 +826,8 < at >< at > static int do_resume(struct dm_ioctl *pa r = dm_swap_table(md, new_map); if (r) { +dm_table_destroy(new_map); dm_put(md); -dm_table_put(new_map); return r; } < at >< at > -835,8 +835,6 < at >< at > static int do_resume(struct dm_ioctl *pa set_disk_ro(dm_disk(md), 0); else set_disk_ro(dm_disk(md), 1); - -dm_table_put(new_map); } if (dm_suspended(md)) < at >< at > -1079,7 +1077,7 < at >< at > static int table_load(struct dm_ioctl *p } if (hc->new_map) -dm_table_put(hc->new_map); +dm_table_destroy(hc->new_map); hc->new_map = t; up_write(&_hash_lock); < at >< at > -1108,7 +1106,7 < at >< at > static int table_clear(struct dm_ioctl * } if (hc->new_map) { -dm_table_put(hc->new_map); +dm_table_destroy(hc->new_map); hc->new_map = NULL; } Index: linux-2.6.24.7-ibmrt2.16-view/drivers/md/dm-table.c =================================================================== --- linux-2.6.24.7-ibmrt2.16-view.orig/drivers/md/dm-table.c +++ linux-2.6.24.7-ibmrt2.16-view/drivers/md/dm-table.c < at >< at > -59,6 +59,19 < at >< at > struct dm_table { void *event_context; }; + /* + * New table reference rules: + * + * The table has always exactly one reference from either mapped_device->map + * or hash_cell->new_map. This reference is not counted in table->holders. + * A pair of dm_create_table/dm_destroy_table functions is used for table + * creation/destruction. + * + * Temporary references from the other code increase table->holders. A pair + * of dm_table_get/dm_table_put functions is used to manipulate it. + * + * When the table is about to be destroyed, we wait for table->holders. + */ /* * Similar to ceiling(log_size(n)) */ < at >< at > -226,7 +239,7 < at >< at > int dm_table_create(struct dm_table **re return -ENOMEM; INIT_LIST_HEAD(&t->devices); -atomic_set(&t->holders, 1); +atomic_set(&t->holders, 0); if (!num_targets) num_targets = KEYS_PER_NODE; < at >< at > -294,10 +307,14 < at >< at > static void free_devices(struct list_hea } } -static void table_destroy(struct dm_table *t) +void dm_table_destroy(struct dm_table *t) { unsigned int i; +while (atomic_read(&t->holders)) +yield(); +smp_mb(); + /* free the indexes (see dm_table_complete) */ if (t->depth >= 2) vfree(t->index[t->depth - 2]); < at >< at > -335,8 +352,9 < at >< at > void dm_table_put(struct dm_table *t) if (!t) return; -if (atomic_dec_and_test(&t->holders)) -table_destroy(t); +smp_mb__before_atomic_dec(); +atomic_dec(&t->holders); + } /* Index: linux-2.6.24.7-ibmrt2.16-view/drivers/md/dm.h =================================================================== --- linux-2.6.24.7-ibmrt2.16-view.orig/drivers/md/dm.h +++ linux-2.6.24.7-ibmrt2.16-view/drivers/md/dm.h < at >< at > -100,6 +100,7 < at >< at > struct dm_table; /*----------------------------------------------------------------- * Internal table functions. *---------------------------------------------------------------*/ +void dm_table_destroy(struct dm_table *t); void dm_table_event_callback(struct dm_table *t, void (*fn)(void *), void *context); struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index);

RE: Re: multipathing pending issues with rhel

Edward Goggin — 2008-11-26T20:13:53

I think I inadvertantly made this matter worse about two years ago when I changed the clariion path checker to issue a scsi read via sg_read in libsg.c after inquiry page 0xc0 in order to discern an inactive from an active clariion snapshot logical unit. This change to multipath-tools/libcheckers/emc_clariion.c causes a path check on a path to incur a reservation error if the reservation is held by a different I/T nexus. A reasonable fix for this problem is to have sg_read in multipath-tools/libcheckers/libsg.c return PATH_UP even if the return value from ioctl is < 0 if the returned scsi status in io_hdr.status is SAM_STAT_RESERVATION_CONFLICT. The dm-mpath.c

Re: [RFC][PATCH 0/4] dm-log: support multi-log devices

Phillip Susi — 2008-11-26T18:56:27

Does it keep track of which log device corresponds to each mirror device and make sure that the log on disk X is updated before the data on disk X? In other words, if you are about to write data to disk 1 that would cause that section to be flagged as dirty, you have to update the log on disk 1 first, not the log on disk 2. The syntax and the fact that the number of logs does not have to equal the number of mirrors makes me think this is not the case.