Bug#840743: mdadm: grow fails when no backup file is specified
Jens Sauer
jsauer65 at gmail.com
Fri Oct 14 12:05:37 UTC 2016
Package: mdadm
Version: 3.3.2-5+deb8u1
Severity: important
Dear Maintainer,
I was playing around with mdadm in a KVM machine.
When the number of active raid devices are changed using the 'grow' feature without
specifying a backup file the reshape fails and the array can not be recovered.
First I created an array:
mdadm -C /dev/md0 -n4 -l6 /dev/vd[bcde]1
Add a spare device:
mdadm --add /dev/md0 /dev/vdf
Grow with a backup-file:
mdadm -G /dev/md0 -n5 --backup-file=/root/grow_md0.bak
Reshape is triggered and was successful.
Add another spare device:
mdadm --add /dev/md0 /dev/vdg1
Grow without backup-file:
mdadm -G /dev/md0 -n6
Syslog shows the following:
md0: detected capacity change from 2142240768 to 3213361152
RAID conf printout:
--- level:6 rd:5 wd:5
disk 0, o:1, dev:vdb1
disk 1, o:1, dev:vdc1
disk 2, o:1, dev:vdd1
disk 3, o:1, dev:vde1
disk 4, o:1, dev:vdf1
RAID conf printout:
--- level:6 rd:6 wd:6
disk 0, o:1, dev:vdb1
disk 1, o:1, dev:vdc1
disk 2, o:1, dev:vdd1
disk 3, o:1, dev:vde1
disk 4, o:1, dev:vdf1
disk 5, o:1, dev:vdg1
md: reshape of RAID array md0
md: minimum _guaranteed_ speed: 1000 KB/sec/disk.
md: using maximum available idle IO bandwidth (but not more than 200000 KB/se
md: using 128k window, over a total of 1046016k.
systemd[1]: Starting Manage MD Reshape on /dev/md0...
systemd[1]: Started Manage MD Reshape on /dev/md0.
systemd[1]: mdadm-grow-continue at md0.service: main process exited, code=exited, status
systemd[1]: Unit mdadm-grow-continue at md0.service entered failed state.
The reshape stucks at 0%, there was no error or warning displayed from mdadm.
I tried to stop and reassemble the array using the --invalid-backup flag but I can't
get it back working.
-- Package-specific info:
--- mdadm.conf
CREATE owner=root group=disk mode=0660 auto=yes
HOMEHOST <system>
MAILADDR root
--- /etc/default/mdadm
INITRDSTART='none'
AUTOCHECK=true
START_DAEMON=true
DAEMON_OPTIONS="--syslog"
VERBOSE=false
--- /proc/mdstat:
Personalities : [raid6] [raid5] [raid4]
md0 : active raid6 vdg1[5] vdf1[4] vde1[3] vdd1[2] vdc1[1] vdb1[0]
3138048 blocks super 1.2 level 6, 512k chunk, algorithm 2 [6/6] [UUUUUU]
[>....................] reshape = 0.0% (0/1046016) finish=25.9min speed=650K/sec
unused devices: <none>
--- /proc/partitions:
major minor #blocks name
254 0 20971520 vda
254 1 20067328 vda1
254 2 1 vda2
254 5 901120 vda5
254 16 1048576 vdb
254 17 1047535 vdb1
254 32 1048576 vdc
254 33 1047535 vdc1
11 0 1048575 sr0
254 48 1048576 vdd
254 49 1047535 vdd1
254 64 1048576 vde
254 65 1047535 vde1
254 80 1048576 vdf
254 81 1047535 vdf1
254 96 1048576 vdg
254 97 1047535 vdg1
254 112 1048576 vdh
254 113 1047535 vdh1
9 0 3138048 md0
--- LVM physical volumes:
LVM does not seem to be used.
--- mount output
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,relatime,size=10240k,nr_inodes=255145,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,relatime,size=411692k,mode=755)
/dev/vda1 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=23,pgrp=1,timeout=300,minproto=5,maxproto=5,direct)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
mqueue on /dev/mqueue type mqueue (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
rpc_pipefs on /run/rpc_pipefs type rpc_pipefs (rw,relatime)
--- initrd.img-3.16.0-4-amd64:
93022 blocks
793809de2771258a23ce068eb898f78c ./lib/modules/3.16.0-4-amd64/kernel/drivers/md/raid0.ko
75eef8213b05ef34ddef822719c540c7 ./lib/modules/3.16.0-4-amd64/kernel/drivers/md/dm-mod.ko
44a8e05dd2b8141ad5c41de91f148e71 ./lib/modules/3.16.0-4-amd64/kernel/drivers/md/md-mod.ko
a995ce07ed228c5ef097e0dc0b496c4a ./lib/modules/3.16.0-4-amd64/kernel/drivers/md/linear.ko
75eacc49b002ab24aeba379671d00ba7 ./lib/modules/3.16.0-4-amd64/kernel/drivers/md/raid10.ko
a02e1b32707dbbe02c3bac85fd9004e1 ./lib/modules/3.16.0-4-amd64/kernel/drivers/md/multipath.ko
c0e7652b9f4e57a9b9e7cba2a96cd8f8 ./lib/modules/3.16.0-4-amd64/kernel/drivers/md/raid1.ko
2c147b2e1a23c28854a5c88c20c68776 ./lib/modules/3.16.0-4-amd64/kernel/drivers/md/raid456.ko
31454ee7ed7b8fe3986ee54f8f31ddab ./conf/mdadm
8e77686a1bbfb047270c4f492a95f4d9 ./etc/mdadm/mdadm.conf
d3be82c0f275d6c25b04d388baf9e836 ./etc/modprobe.d/mdadm.conf
131be8e959b59b6dc48c289966125cd4 ./sbin/mdadm
599bbf3fe6093157a26863dcb59cdf5d ./scripts/local-top/mdadm
--- initrd's /conf/conf.d/md:
no conf/md file.
--- /proc/modules:
raid456 77553 1 - Live 0xffffffffa01d7000
async_raid6_recov 16626 1 raid456, Live 0xffffffffa01d1000
async_memcpy 12394 2 raid456,async_raid6_recov, Live 0xffffffffa0158000
async_pq 12561 2 raid456,async_raid6_recov, Live 0xffffffffa0129000
async_xor 12429 3 raid456,async_raid6_recov,async_pq, Live 0xffffffffa0124000
async_tx 12566 5 raid456,async_raid6_recov,async_memcpy,async_pq,async_xor, Live 0xffffffffa00ae000
raid6_pq 95238 2 async_raid6_recov,async_pq, Live 0xffffffffa008a000
md_mod 107672 1 raid456, Live 0xffffffffa019b000
--- /var/log/syslog:
--- volume detail:
/dev/vda:
MBR Magic : aa55
Partition[0] : 40134656 sectors at 2048 (type 83)
Partition[1] : 1802242 sectors at 40138750 (type 05)
--
/dev/vda1 is not recognised by mdadm.
/dev/vda2:
MBR Magic : aa55
Partition[0] : 1802240 sectors at 2 (type 82)
--
/dev/vda5 is not recognised by mdadm.
/dev/vdb:
MBR Magic : aa55
Partition[0] : 2097151 sectors at 1 (type ee)
--
/dev/vdb1:
Magic : a92b4efc
Version : 1.2
Feature Map : 0x4
Array UUID : d0bec14c:1e9d8870:dc23e0e4:c158a55c
Name : debian:0 (local to host debian)
Creation Time : Fri Oct 14 13:04:04 2016
Raid Level : raid6
Raid Devices : 6
Avail Dev Size : 2093023 (1022.16 MiB 1071.63 MB)
Array Size : 4184064 (3.99 GiB 4.28 GB)
Used Dev Size : 2092032 (1021.67 MiB 1071.12 MB)
Data Offset : 2048 sectors
Super Offset : 8 sectors
Unused Space : before=1960 sectors, after=991 sectors
State : clean
Device UUID : 1374c6e8:3e29a424:6fa3436c:faa0e082
Reshape pos'n : 0
Delta Devices : 1 (5->6)
Update Time : Fri Oct 14 13:06:20 2016
Bad Block Log : 512 entries available at offset 72 sectors
Checksum : ee3c41a0 - correct
Events : 49
Layout : left-symmetric
Chunk Size : 512K
Device Role : Active device 0
Array State : AAAAAA ('A' == active, '.' == missing, 'R' == replacing)
--
/dev/vdc:
MBR Magic : aa55
Partition[0] : 2097151 sectors at 1 (type ee)
--
/dev/vdc1:
Magic : a92b4efc
Version : 1.2
Feature Map : 0x4
Array UUID : d0bec14c:1e9d8870:dc23e0e4:c158a55c
Name : debian:0 (local to host debian)
Creation Time : Fri Oct 14 13:04:04 2016
Raid Level : raid6
Raid Devices : 6
Avail Dev Size : 2093023 (1022.16 MiB 1071.63 MB)
Array Size : 4184064 (3.99 GiB 4.28 GB)
Used Dev Size : 2092032 (1021.67 MiB 1071.12 MB)
Data Offset : 2048 sectors
Super Offset : 8 sectors
Unused Space : before=1960 sectors, after=991 sectors
State : clean
Device UUID : e8a5457e:d4ab2422:92901a20:a1533853
Reshape pos'n : 0
Delta Devices : 1 (5->6)
Update Time : Fri Oct 14 13:06:20 2016
Bad Block Log : 512 entries available at offset 72 sectors
Checksum : 56a95d6 - correct
Events : 49
Layout : left-symmetric
Chunk Size : 512K
Device Role : Active device 1
Array State : AAAAAA ('A' == active, '.' == missing, 'R' == replacing)
--
/dev/vdd:
MBR Magic : aa55
Partition[0] : 2097151 sectors at 1 (type ee)
--
/dev/vdd1:
Magic : a92b4efc
Version : 1.2
Feature Map : 0x4
Array UUID : d0bec14c:1e9d8870:dc23e0e4:c158a55c
Name : debian:0 (local to host debian)
Creation Time : Fri Oct 14 13:04:04 2016
Raid Level : raid6
Raid Devices : 6
Avail Dev Size : 2093023 (1022.16 MiB 1071.63 MB)
Array Size : 4184064 (3.99 GiB 4.28 GB)
Used Dev Size : 2092032 (1021.67 MiB 1071.12 MB)
Data Offset : 2048 sectors
Super Offset : 8 sectors
Unused Space : before=1960 sectors, after=991 sectors
State : clean
Device UUID : 694a5fde:0a3c3ee2:f56c990c:e12c1fe2
Reshape pos'n : 0
Delta Devices : 1 (5->6)
Update Time : Fri Oct 14 13:06:20 2016
Bad Block Log : 512 entries available at offset 72 sectors
Checksum : a1038032 - correct
Events : 49
Layout : left-symmetric
Chunk Size : 512K
Device Role : Active device 2
Array State : AAAAAA ('A' == active, '.' == missing, 'R' == replacing)
--
/dev/vde:
MBR Magic : aa55
Partition[0] : 2097151 sectors at 1 (type ee)
--
/dev/vde1:
Magic : a92b4efc
Version : 1.2
Feature Map : 0x4
Array UUID : d0bec14c:1e9d8870:dc23e0e4:c158a55c
Name : debian:0 (local to host debian)
Creation Time : Fri Oct 14 13:04:04 2016
Raid Level : raid6
Raid Devices : 6
Avail Dev Size : 2093023 (1022.16 MiB 1071.63 MB)
Array Size : 4184064 (3.99 GiB 4.28 GB)
Used Dev Size : 2092032 (1021.67 MiB 1071.12 MB)
Data Offset : 2048 sectors
Super Offset : 8 sectors
Unused Space : before=1960 sectors, after=991 sectors
State : clean
Device UUID : 871abe17:ffa0c98b:e77fc7e1:4fa7bdff
Reshape pos'n : 0
Delta Devices : 1 (5->6)
Update Time : Fri Oct 14 13:06:20 2016
Bad Block Log : 512 entries available at offset 72 sectors
Checksum : 76ba42a6 - correct
Events : 49
Layout : left-symmetric
Chunk Size : 512K
Device Role : Active device 3
Array State : AAAAAA ('A' == active, '.' == missing, 'R' == replacing)
--
/dev/vdf:
MBR Magic : aa55
Partition[0] : 2097151 sectors at 1 (type ee)
--
/dev/vdf1:
Magic : a92b4efc
Version : 1.2
Feature Map : 0x4
Array UUID : d0bec14c:1e9d8870:dc23e0e4:c158a55c
Name : debian:0 (local to host debian)
Creation Time : Fri Oct 14 13:04:04 2016
Raid Level : raid6
Raid Devices : 6
Avail Dev Size : 2093023 (1022.16 MiB 1071.63 MB)
Array Size : 4184064 (3.99 GiB 4.28 GB)
Used Dev Size : 2092032 (1021.67 MiB 1071.12 MB)
Data Offset : 2048 sectors
Super Offset : 8 sectors
Unused Space : before=1960 sectors, after=991 sectors
State : clean
Device UUID : 198a5e32:0b55170e:aac39def:1d62089a
Reshape pos'n : 0
Delta Devices : 1 (5->6)
Update Time : Fri Oct 14 13:06:20 2016
Bad Block Log : 512 entries available at offset 72 sectors
Checksum : bbc964d5 - correct
Events : 49
Layout : left-symmetric
Chunk Size : 512K
Device Role : Active device 4
Array State : AAAAAA ('A' == active, '.' == missing, 'R' == replacing)
--
/dev/vdg:
MBR Magic : aa55
Partition[0] : 2097151 sectors at 1 (type ee)
--
/dev/vdg1:
Magic : a92b4efc
Version : 1.2
Feature Map : 0x4
Array UUID : d0bec14c:1e9d8870:dc23e0e4:c158a55c
Name : debian:0 (local to host debian)
Creation Time : Fri Oct 14 13:04:04 2016
Raid Level : raid6
Raid Devices : 6
Avail Dev Size : 2093023 (1022.16 MiB 1071.63 MB)
Array Size : 4184064 (3.99 GiB 4.28 GB)
Used Dev Size : 2092032 (1021.67 MiB 1071.12 MB)
Data Offset : 2048 sectors
Super Offset : 8 sectors
Unused Space : before=1960 sectors, after=991 sectors
State : clean
Device UUID : e2821054:b6d9d287:d0108c45:5945d6e7
Reshape pos'n : 0
Delta Devices : 1 (5->6)
Update Time : Fri Oct 14 13:06:20 2016
Bad Block Log : 512 entries available at offset 72 sectors
Checksum : faf312ac - correct
Events : 49
Layout : left-symmetric
Chunk Size : 512K
Device Role : Active device 5
Array State : AAAAAA ('A' == active, '.' == missing, 'R' == replacing)
--
/dev/vdh:
MBR Magic : aa55
Partition[0] : 2097151 sectors at 1 (type ee)
--
/dev/vdh1 is not recognised by mdadm.
--- /proc/cmdline
BOOT_IMAGE=/boot/vmlinuz-3.16.0-4-amd64 root=UUID=ce33ccf9-2816-4829-96db-9d48bc741656 ro quiet
--- grub2:
linux /boot/vmlinuz-3.16.0-4-amd64 root=UUID=ce33ccf9-2816-4829-96db-9d48bc741656 ro quiet
linux /boot/vmlinuz-3.16.0-4-amd64 root=UUID=ce33ccf9-2816-4829-96db-9d48bc741656 ro quiet
linux /boot/vmlinuz-3.16.0-4-amd64 root=UUID=ce33ccf9-2816-4829-96db-9d48bc741656 ro single
--- udev:
ii udev 215-17+deb8u amd64 /dev/ and hotplug management daem
e95faba71945b1dcc9749adc5dcd80f6 /lib/udev/rules.d/63-md-raid-arrays.rules
f0b45148fbd3ce678e682925483f17dc /lib/udev/rules.d/64-md-raid-assembly.rules
13e82ea039db387bce78de8e7aeb2f34 /lib/udev/rules.d/99-systemd.rules
--- /dev:
brw-rw---- 1 root disk 9, 0 Oct 14 13:04 /dev/md0
/dev/disk/by-id:
total 0
lrwxrwxrwx 1 root root 9 Oct 14 11:34 ata-QEMU_DVD-ROM_QM00001 -> ../../sr0
lrwxrwxrwx 1 root root 9 Oct 14 13:04 md-name-debian:0 -> ../../md0
lrwxrwxrwx 1 root root 9 Oct 14 13:04 md-uuid-d0bec14c:1e9d8870:dc23e0e4:c158a55c -> ../../md0
/dev/disk/by-partuuid:
total 0
lrwxrwxrwx 1 root root 10 Oct 14 13:04 17f80132-5294-4800-89c0-1601e3261060 -> ../../vdf1
lrwxrwxrwx 1 root root 10 Oct 14 13:04 243861e8-c20e-4c61-aea1-bccf3e001321 -> ../../vde1
lrwxrwxrwx 1 root root 10 Oct 14 13:06 5cee693e-8da6-4820-9b79-49543c400cad -> ../../vdg1
lrwxrwxrwx 1 root root 10 Oct 14 13:04 6a00d69e-8f34-4c39-aa0e-a83dc2f48a46 -> ../../vdd1
lrwxrwxrwx 1 root root 10 Oct 14 13:04 83082b5f-54c2-4435-b7da-3e16f4aa5182 -> ../../vdc1
lrwxrwxrwx 1 root root 10 Oct 14 13:04 9e83ed54-12ca-4854-812c-2e7da7d2906b -> ../../vdb1
lrwxrwxrwx 1 root root 10 Oct 14 13:03 a5eef62e-0fb0-4c45-ad2d-245c7a352c63 -> ../../vdh1
/dev/disk/by-uuid:
total 0
lrwxrwxrwx 1 root root 10 Oct 14 11:34 ace0aaed-cef7-4988-acbd-21e8bdb73c06 -> ../../vda5
lrwxrwxrwx 1 root root 10 Oct 14 11:34 ce33ccf9-2816-4829-96db-9d48bc741656 -> ../../vda1
Auto-generated on Fri, 14 Oct 2016 13:33:10 +0200 by mdadm bugscript
-- System Information:
Debian Release: 8.6
APT prefers stable-updates
APT policy: (500, 'stable-updates'), (500, 'stable')
Architecture: amd64 (x86_64)
Kernel: Linux 3.16.0-4-amd64 (SMP w/2 CPU cores)
Locale: LANG=de_DE.UTF-8, LC_CTYPE=de_DE.UTF-8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash
Init: systemd (via /run/systemd/system)
Versions of packages mdadm depends on:
ii debconf [debconf-2.0] 1.5.56
ii initscripts 2.88dsf-59
ii libc6 2.19-18+deb8u6
ii lsb-base 4.1+Debian13+nmu1
ii udev 215-17+deb8u5
Versions of packages mdadm recommends:
ii exim4-daemon-light [mail-transport-agent] 4.84.2-2+deb8u1
ii kmod 18-3
mdadm suggests no packages.
-- debconf information:
mdadm/mail_to: root
mdadm/initrdstart_msg_errmd:
* mdadm/initrdstart: none
mdadm/start_daemon: true
mdadm/initrdstart_notinconf: false
mdadm/initrdstart_msg_errexist:
mdadm/initrdstart_msg_errconf:
mdadm/initrdstart_msg_intro:
mdadm/initrdstart_msg_errblock:
mdadm/autocheck: true
More information about the pkg-mdadm-devel
mailing list