物理机断电导致osd故障排查记录

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

ceph -s
  cluster:
    id:     baf87797-3ec1-4f2c-8126-bf0a44051b13
    health: HEALTH_WARN
            3 osds down
            1 host (3 osds) down
            1 pools have many more objects per pg than average
            Degraded data redundancy: 1167403/4841062 objects degraded (24.115%), 391 pgs degraded, 412 pgs undersized
 
  services:
    mon: 3 daemons, quorum 10.10.20.4,10.10.20.123,10.10.20.186 (age 9m)
    mgr: 10.10.20.4(active, since 9m), standbys: 10.10.20.123, 10.10.20.186
    mds: kubefs:2 {0=10.10.20.123=up:active,1=10.10.20.186=up:active} 1 up:standby
    osd: 13 osds: 10 up (since 11m), 13 in (since 14M)
    rgw: 4 daemons active (10.10.20.4, 10.10.20.8, 10.10.20.123, 10.10.20.186)
 
  data:
    pools:   9 pools, 896 pgs
    objects: 2.42M objects, 1.8 TiB
    usage:   2.8 TiB used, 4.0 TiB / 6.8 TiB avail
    pgs:     1167403/4841062 objects degraded (24.115%)
             484 active+clean
             391 active+undersized+degraded
             21  active+undersized
 
  io:
    client:   1.6 MiB/s rd, 303 KiB/s wr, 6 op/s rd, 22 op/s wr

1
2
3
4
5
6
7
8

# ceph health detail
HEALTH_WARN 3 osds down; 1 host (3 osds) down; 1 pools have many more objects per pg than average; Degraded data redundancy: 1167404/4841068 objects degraded (24.115%), 391 pgs degraded, 412 pgs undersized
OSD_DOWN 3 osds down
    osd.2 (root=default,host=10.10.20.4) is down
    osd.3 (root=default,host=10.10.20.4) is down
    osd.6 (root=default,host=10.10.20.4) is down
OSD_HOST_DOWN 1 host (3 osds) down
    host 10.10.20.4 (root=default) (3 osds) is down

1
2
3
4
5
6
7
8
9

$ ps -ef|grep ceph
root        1260       1  0 15:21 ?        00:00:00 /usr/bin/python2.7 /usr/bin/ceph-crash
ceph        1858       1  0 15:21 ?        00:00:00 /usr/bin/ceph-mds -f --cluster ceph --id 10.10.20.4 --setuser ceph --setgroup ceph
ceph        1861       1  0 15:21 ?        00:00:01 /usr/bin/radosgw -f --cluster ceph --name client.rgw.10.10.20.4 --setuser ceph --setgroup ceph
ceph        1872       1  0 15:21 ?        00:00:06 /usr/bin/ceph-mon -f --cluster ceph --id 10.10.20.4 --setuser ceph --setgroup ceph
ceph        1917       1  6 15:21 ?        00:00:37 /usr/bin/ceph-mgr -f --cluster ceph --id 10.10.20.4 --setuser ceph --setgroup ceph
root        1944       2  0 15:21 ?        00:00:00 [ceph-msgr]
root        1994       2  0 15:21 ?        00:00:00 [ceph-watch-noti]
root        9095    8488  0 15:31 pts/0    00:00:00 grep --color=auto ceph	

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

$ systemctl status ceph-osd@2.service        
● ceph-osd@2.service - Ceph object storage daemon osd.2
   Loaded: loaded (/usr/lib/systemd/system/ceph-osd@.service; enabled-runtime; vendor preset: disabled)
   Active: failed (Result: start-limit) since Wed 2025-04-16 15:56:31 +08; 1s ago
  Process: 14689 ExecStart=/usr/bin/ceph-osd -f --cluster ${CLUSTER} --id %i --setuser ceph --setgroup ceph (code=exited, status=5)
  Process: 14684 ExecStartPre=/usr/lib/ceph/ceph-osd-prestart.sh --cluster ${CLUSTER} --id %i (code=exited, status=0/SUCCESS)
 Main PID: 14689 (code=exited, status=5)

Apr 16 15:56:31 10.10.20.4 systemd[1]: Unit ceph-osd@2.service entered failed state.
Apr 16 15:56:31 10.10.20.4 systemd[1]: ceph-osd@2.service failed.
Apr 16 15:56:31 10.10.20.4 systemd[1]: ceph-osd@2.service holdoff time over, scheduling restart.
Apr 16 15:56:31 10.10.20.4 systemd[1]: Stopped Ceph object storage daemon osd.2.
Apr 16 15:56:31 10.10.20.4 systemd[1]: start request repeated too quickly for ceph-osd@2.service
Apr 16 15:56:31 10.10.20.4 systemd[1]: Failed to start Ceph object storage daemon osd.2.
Apr 16 15:56:31 10.10.20.4 systemd[1]: Unit ceph-osd@2.service entered failed state.
Apr 16 15:56:31 10.10.20.4 systemd[1]: ceph-osd@2.service failed.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

$ tail -200 /var/log/ceph/ceph-osd.2.log

  cache_index_and_filter_blocks: 1
  cache_index_and_filter_blocks_with_high_priority: 1
  pin_l0_filter_and_index_blocks_in_cache: 1
  pin_top_level_index_and_filter: 1
  index_type: 0
  data_block_index_type: 0
  data_block_hash_table_util_ratio: 0.750000
  hash_index_allow_collision: 1
  checksum: 1
  no_block_cache: 0
  block_cache: 0x560db5e76e50
  block_cache_name: BinnedLRUCache
  block_cache_options:
    capacity : 429496729
    num_shard_bits : 4
    strict_capacity_limit : 0
    high_pri_pool_ratio: 0.000
  block_cache_compressed: (nil)
  persistent_cache: (nil)
  block_size: 4096
  block_size_deviation: 10
  block_restart_interval: 16
  index_block_restart_interval: 1
  metadata_block_size: 4096
  partition_filters: 0
  use_delta_encoding: 1
  filter_policy: rocksdb.BuiltinBloomFilter
  whole_key_filtering: 1
  verify_compression: 0
  read_amp_bytes_per_bit: 0
  format_version: 2
  enable_index_compression: 1
  block_align: 0

   -74> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:        Options.write_buffer_size: 268435456
   -73> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:  Options.max_write_buffer_number: 4
   -72> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:          Options.compression: NoCompression
   -71> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                  Options.bottommost_compression: Disabled
   -70> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:       Options.prefix_extractor: nullptr
   -69> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:   Options.memtable_insert_with_hint_prefix_extractor: nullptr
   -68> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:             Options.num_levels: 7
   -67> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:        Options.min_write_buffer_number_to_merge: 1
   -66> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:     Options.max_write_buffer_number_to_maintain: 0
   -65> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:            Options.bottommost_compression_opts.window_bits: -14
   -64> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                  Options.bottommost_compression_opts.level: 32767
   -63> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:               Options.bottommost_compression_opts.strategy: 0
   -62> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:         Options.bottommost_compression_opts.max_dict_bytes: 0
   -61> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:         Options.bottommost_compression_opts.zstd_max_train_bytes: 0
   -60> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                  Options.bottommost_compression_opts.enabled: false
   -59> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:            Options.compression_opts.window_bits: -14
   -58> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                  Options.compression_opts.level: 32767
   -57> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:               Options.compression_opts.strategy: 0
   -56> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:         Options.compression_opts.max_dict_bytes: 0
   -55> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:         Options.compression_opts.zstd_max_train_bytes: 0
   -54> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                  Options.compression_opts.enabled: false
   -53> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:      Options.level0_file_num_compaction_trigger: 4
   -52> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:          Options.level0_slowdown_writes_trigger: 20
   -51> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:              Options.level0_stop_writes_trigger: 36
   -50> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                   Options.target_file_size_base: 67108864
   -49> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:             Options.target_file_size_multiplier: 1
   -48> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                Options.max_bytes_for_level_base: 268435456
   -47> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0
   -46> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:          Options.max_bytes_for_level_multiplier: 10.000000
   -45> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1
   -44> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1
   -43> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1
   -42> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1
   -41> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1
   -40> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1
   -39> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1
   -38> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:       Options.max_sequential_skip_in_iterations: 8
   -37> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                    Options.max_compaction_bytes: 1677721600
   -36> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:                        Options.arena_block_size: 33554432
   -35> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:   Options.soft_pending_compaction_bytes_limit: 68719476736
   -34> 2025-04-16 15:56:30.913 7ffbbdfbfdc0  4 rocksdb:   Options.hard_pending_compaction_bytes_limit: 274877906944
   -33> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:       Options.rate_limit_delay_max_milliseconds: 100
   -32> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                Options.disable_auto_compactions: 0
   -31> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                        Options.compaction_style: kCompactionStyleLevel
   -30> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                          Options.compaction_pri: kMinOverlappingRatio
   -29> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb: Options.compaction_options_universal.size_ratio: 1
   -28> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb: Options.compaction_options_universal.min_merge_width: 2
   -27> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295
   -26> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200
   -25> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1
   -24> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
   -23> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824
   -22> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0
   -21> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                   Options.table_properties_collectors: 
   -20> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                   Options.inplace_update_support: 0
   -19> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                 Options.inplace_update_num_locks: 10000
   -18> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:               Options.memtable_prefix_bloom_size_ratio: 0.000000
   -17> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:               Options.memtable_whole_key_filtering: 0
   -16> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:   Options.memtable_huge_page_size: 0
   -15> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                           Options.bloom_locality: 0
   -14> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                    Options.max_successive_merges: 0
   -13> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                Options.optimize_filters_for_hits: 0
   -12> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                Options.paranoid_file_checks: 0
   -11> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                Options.force_consistency_checks: 0
   -10> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                Options.report_bg_io_stats: 0
    -9> 2025-04-16 15:56:30.914 7ffbbdfbfdc0  4 rocksdb:                               Options.ttl: 0
    -8> 2025-04-16 15:56:30.919 7ffbbdfbfdc0  4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-002160 succeeded,manifest_file_number is 2160, next_file_number is 8221, last_sequence is 433449593, log_number is 8218,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0

    -7> 2025-04-16 15:56:30.919 7ffbbdfbfdc0  4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 8218

    -6> 2025-04-16 15:56:30.919 7ffbbdfbfdc0  4 rocksdb: EVENT_LOG_v1 {"time_micros": 1744790190920696, "job": 1, "event": "recovery_started", "log_files": [8216, 8218]}
    -5> 2025-04-16 15:56:30.919 7ffbbdfbfdc0  4 rocksdb: [db/db_impl_open.cc:583] Recovering log #8216 mode 0
    -4> 2025-04-16 15:56:31.035 7ffbbdfbfdc0  4 rocksdb: [db/db_impl_open.cc:583] Recovering log #8218 mode 0
    -3> 2025-04-16 15:56:31.321 7ffbbdfbfdc0 -1 bdev(0x560db6af5500 /var/lib/ceph/osd/ceph-2/block) _sync_write sync_file_range error: (5) Input/output error
    -2> 2025-04-16 15:56:31.385 7ffbae5ac700 -1 bdev(0x560db6af5500 /var/lib/ceph/osd/ceph-2/block) _aio_thread got r=-5 ((5) Input/output error)
    -1> 2025-04-16 15:56:31.387 7ffbae5ac700 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14.2.4/rpm/el7/BUILD/ceph-14.2.4/src/os/bluestore/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7ffbae5ac700 time 2025-04-16 15:56:31.386670
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14.2.4/rpm/el7/BUILD/ceph-14.2.4/src/os/bluestore/KernelDevice.cc: 534: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!")

 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
 1: (ceph::__ceph_abort(char const*, int, char const*, std::string const&)+0xdd) [0x560dab22909d]
 2: (KernelDevice::_aio_thread()+0xca8) [0x560dab869918]
 3: (KernelDevice::AioCompletionThread::entry()+0xd) [0x560dab86b07d]
 4: (()+0x7e65) [0x7ffbbabf9e65]
 5: (clone()+0x6d) [0x7ffbb9abd88d]

     0> 2025-04-16 15:56:31.389 7ffbae5ac700 -1 *** Caught signal (Aborted) **
 in thread 7ffbae5ac700 thread_name:bstore_aio

 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
 1: (()+0xf5f0) [0x7ffbbac015f0]
 2: (gsignal()+0x37) [0x7ffbb99f5337]
 3: (abort()+0x148) [0x7ffbb99f6a28]
 4: (ceph::__ceph_abort(char const*, int, char const*, std::string const&)+0x1a5) [0x560dab229165]
 5: (KernelDevice::_aio_thread()+0xca8) [0x560dab869918]
 6: (KernelDevice::AioCompletionThread::entry()+0xd) [0x560dab86b07d]
 7: (()+0x7e65) [0x7ffbbabf9e65]
 8: (clone()+0x6d) [0x7ffbb9abd88d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

1
2

    -3> 2025-04-16 15:56:31.321 7ffbbdfbfdc0 -1 bdev(0x560db6af5500 /var/lib/ceph/osd/ceph-2/block) _sync_write sync_file_range error: (5) Input/output error
    -2> 2025-04-16 15:56:31.385 7ffbae5ac700 -1 bdev(0x560db6af5500 /var/lib/ceph/osd/ceph-2/block) _aio_thread got r=-5 ((5) Input/output error)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

#lsblk
NAME                                                                                                  MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda                                                                                                     8:0    0  200G  0 disk 
└─sda1                                                                                                  8:1    0  200G  0 part /
sdb                                                                                                     8:16   0  700G  0 disk 
└─ceph--4834f9df--5dd2--434c--ac3b--5da26b702766-osd--block--40ce0a60--8b70--4fc9--b160--3d2f27ca51a0 253:2    0  699G  0 lvm  
sdc                                                                                                     8:32   0  700G  0 disk 
└─ceph--19a5530a--d904--4373--9c06--5de4a2aedb4c-osd--block--a9765670--aab3--47ed--a849--307bef9c2c13 253:0    0  699G  0 lvm  
sdd                                                                                                     8:48   0  700G  0 disk 
└─ceph--cfc9f372--53c3--4646--8661--0f23e873baf2-osd--block--a0cde5de--a2a4--43ae--9c7b--71e7cec2bdfd 253:1    0  699G  0 lvm  
sde                                                                                                     8:64   0  100G  0 disk 
sdf  

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

sudo -u root ceph-volume lvm list	



====== osd.2 =======

  [block]       /dev/ceph-4834f9df-5dd2-434c-ac3b-5da26b702766/osd-block-40ce0a60-8b70-4fc9-b160-3d2f27ca51a0

      block device              /dev/ceph-4834f9df-5dd2-434c-ac3b-5da26b702766/osd-block-40ce0a60-8b70-4fc9-b160-3d2f27ca51a0
      block uuid                5Dg5lG-CtP9-jf0H-QJGD-DXuS-cWym-b2IwCW
      cephx lockbox secret      
      cluster fsid              baf87797-3ec1-4f2c-8126-bf0a44051b13
      cluster name              ceph
      crush device class        None
      encrypted                 0
      osd fsid                  40ce0a60-8b70-4fc9-b160-3d2f27ca51a0
      osd id                    2
      type                      block
      vdo                       0
      devices                   /dev/sdb

====== osd.3 =======

  [block]       /dev/ceph-cfc9f372-53c3-4646-8661-0f23e873baf2/osd-block-a0cde5de-a2a4-43ae-9c7b-71e7cec2bdfd

      block device              /dev/ceph-cfc9f372-53c3-4646-8661-0f23e873baf2/osd-block-a0cde5de-a2a4-43ae-9c7b-71e7cec2bdfd
      block uuid                cJ7oS3-olgL-U6PM-3WkY-wePU-4bY4-FpqYOO
      cephx lockbox secret      
      cluster fsid              baf87797-3ec1-4f2c-8126-bf0a44051b13
      cluster name              ceph
      crush device class        None
      encrypted                 0
      osd fsid                  a0cde5de-a2a4-43ae-9c7b-71e7cec2bdfd
      osd id                    3
      type                      block
      vdo                       0
      devices                   /dev/sdd

====== osd.6 =======

  [block]       /dev/ceph-19a5530a-d904-4373-9c06-5de4a2aedb4c/osd-block-a9765670-aab3-47ed-a849-307bef9c2c13

      block device              /dev/ceph-19a5530a-d904-4373-9c06-5de4a2aedb4c/osd-block-a9765670-aab3-47ed-a849-307bef9c2c13
      block uuid                OH3Fv2-ebt0-E1qA-R347-f7G6-fdeW-28OwhK
      cephx lockbox secret      
      cluster fsid              baf87797-3ec1-4f2c-8126-bf0a44051b13
      cluster name              ceph
      crush device class        None
      encrypted                 0
      osd fsid                  a9765670-aab3-47ed-a849-307bef9c2c13
      osd id                    6
      type                      block
      vdo                       0
      devices                   /dev/sdc

1
2

#wipefs -a /dev/sdb 
wipefs: error: /dev/sdb: probing initialization failed: Device or resource busy

1
2
3
4
5
6

ceph osd set noout 
ceph osd out osd.2
ceph -w
ceph osd crush remove osd.2
ceph osd rm osd.2
ceph osd df