diff options
Diffstat (limited to '0406-afr-support-split-brain-CLI-for-replica-3.patch')
| -rw-r--r-- | 0406-afr-support-split-brain-CLI-for-replica-3.patch | 185 |
1 files changed, 185 insertions, 0 deletions
diff --git a/0406-afr-support-split-brain-CLI-for-replica-3.patch b/0406-afr-support-split-brain-CLI-for-replica-3.patch new file mode 100644 index 0000000..4b57e8a --- /dev/null +++ b/0406-afr-support-split-brain-CLI-for-replica-3.patch @@ -0,0 +1,185 @@ +From a75bb15fbe64f14580c44b8a33314c8bbeffdede Mon Sep 17 00:00:00 2001 +From: Ravishankar N <ravishankar@redhat.com> +Date: Thu, 4 Jun 2020 18:54:46 +0530 +Subject: [PATCH 406/449] afr: support split-brain CLI for replica 3 + +Patch in upstream master: https://review.gluster.org/#/c/glusterfs/+/23502/ + +Ever since we added quorum checks for lookups in afr via commit +bd44d59741bb8c0f5d7a62c5b1094179dd0ce8a4, the split-brain resolution +commands would not work for replica 3 because there would be no +readables for the lookup fop. + +The argument was that split-brains do not occur in replica 3 but we do +see (data/metadata) split-brain cases once in a while which indicate that there are +a few bugs/corner cases yet to be discovered and fixed. + +Fortunately, commit 8016d51a3bbd410b0b927ed66be50a09574b7982 added +GF_CLIENT_PID_GLFS_HEALD as the pid for all fops made by glfsheal. If we +leverage this and allow lookups in afr when pid is GF_CLIENT_PID_GLFS_HEALD, +split-brain resolution commands will work for replica 3 volumes too. + +Likewise, the check is added in shard_lookup as well to permit resolving +split-brains by specifying "/.shard/shard-file.xx" as the file name +(which previously used to fail with EPERM). + +BUG: 1759875 +Change-Id: I203735b909c7d30fc4faaf3ecd4f5b6b379ab266 +Signed-off-by: Ravishankar N <ravishankar@redhat.com> +Reviewed-on: https://code.engineering.redhat.com/gerrit/202375 +Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com> +--- + .../replicate/bug-1756938-replica-3-sbrain-cli.t | 111 +++++++++++++++++++++ + xlators/cluster/afr/src/afr-common.c | 3 +- + xlators/features/shard/src/shard.c | 3 +- + 3 files changed, 115 insertions(+), 2 deletions(-) + create mode 100644 tests/bugs/replicate/bug-1756938-replica-3-sbrain-cli.t + +diff --git a/tests/bugs/replicate/bug-1756938-replica-3-sbrain-cli.t b/tests/bugs/replicate/bug-1756938-replica-3-sbrain-cli.t +new file mode 100644 +index 0000000..c1bdf34 +--- /dev/null ++++ b/tests/bugs/replicate/bug-1756938-replica-3-sbrain-cli.t +@@ -0,0 +1,111 @@ ++#!/bin/bash ++ ++. $(dirname $0)/../../include.rc ++. $(dirname $0)/../../volume.rc ++. $(dirname $0)/../../afr.rc ++ ++cleanup; ++ ++TEST glusterd; ++TEST pidof glusterd; ++TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2} ++TEST $CLI volume set $V0 features.shard enable ++TEST $CLI volume set $V0 features.shard-block-size 4MB ++ ++TEST $CLI volume start $V0 ++EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 ++EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1 ++EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2 ++TEST glusterfs --volfile-server=$H0 --volfile-id=/$V0 $M0 ++EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0 ++EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1 ++EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 2 ++ ++#Create split-brain by setting afr xattrs/gfids manually. ++#file1 is non-sharded and will be in data split-brain. ++#file2 will have one shard which will be in data split-brain. ++#file3 will have one shard which will be in gfid split-brain. ++#file4 will have one shard which will be in data & metadata split-brain. ++TEST dd if=/dev/zero of=$M0/file1 bs=1024 count=1024 oflag=direct ++TEST dd if=/dev/zero of=$M0/file2 bs=1M count=6 oflag=direct ++TEST dd if=/dev/zero of=$M0/file3 bs=1M count=6 oflag=direct ++TEST dd if=/dev/zero of=$M0/file4 bs=1M count=6 oflag=direct ++EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 ++ ++#------------------------------------------------------------------------------- ++TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/${V0}0/file1 ++TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000000 $B0/${V0}0/file1 ++TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000000 $B0/${V0}1/file1 ++TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000000 $B0/${V0}1/file1 ++TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000000 $B0/${V0}2/file1 ++TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/${V0}2/file1 ++ ++#------------------------------------------------------------------------------- ++gfid_f2=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/file2)) ++TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/${V0}0/.shard/$gfid_f2.1 ++TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000000 $B0/${V0}0/.shard/$gfid_f2.1 ++TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000000 $B0/${V0}1/.shard/$gfid_f2.1 ++TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000000 $B0/${V0}1/.shard/$gfid_f2.1 ++TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000000 $B0/${V0}2/.shard/$gfid_f2.1 ++TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/${V0}2/.shard/$gfid_f2.1 ++ ++#------------------------------------------------------------------------------- ++TESTS_EXPECTED_IN_LOOP=5 ++function assign_new_gfid { ++ brickpath=$1 ++ filename=$2 ++ gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $brickpath/$filename)) ++ gfid_shard=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $brickpath/.shard/$gfid.1)) ++ ++ TEST rm $brickpath/.glusterfs/${gfid_shard:0:2}/${gfid_shard:2:2}/$gfid_shard ++ TEST setfattr -x trusted.gfid $brickpath/.shard/$gfid.1 ++ new_gfid=$(get_random_gfid) ++ new_gfid_str=$(gf_gfid_xattr_to_str $new_gfid) ++ TEST setfattr -n trusted.gfid -v $new_gfid $brickpath/.shard/$gfid.1 ++ TEST mkdir -p $brickpath/.glusterfs/${new_gfid_str:0:2}/${new_gfid_str:2:2} ++ TEST ln $brickpath/.shard/$gfid.1 $brickpath/.glusterfs/${new_gfid_str:0:2}/${new_gfid_str:2:2}/$new_gfid_str ++} ++assign_new_gfid $B0/$V0"1" file3 ++assign_new_gfid $B0/$V0"2" file3 ++ ++#------------------------------------------------------------------------------- ++gfid_f4=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/file4)) ++TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000100000000 $B0/${V0}0/.shard/$gfid_f4.1 ++TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000100000000 $B0/${V0}0/.shard/$gfid_f4.1 ++TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000100000000 $B0/${V0}1/.shard/$gfid_f4.1 ++TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000100000000 $B0/${V0}1/.shard/$gfid_f4.1 ++TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000100000000 $B0/${V0}2/.shard/$gfid_f4.1 ++TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000100000000 $B0/${V0}2/.shard/$gfid_f4.1 ++ ++#------------------------------------------------------------------------------- ++#Add entry to xattrop dir on first brick and check for split-brain. ++xattrop_dir0=$(afr_get_index_path $B0/$V0"0") ++base_entry_b0=`ls $xattrop_dir0` ++ ++gfid_f1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/file1)) ++TEST ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_f1 ++ ++gfid_f2_shard1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/.shard/$gfid_f2.1)) ++TEST ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_f2_shard1 ++ ++gfid_f3=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/file3)) ++gfid_f3_shard1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/.shard/$gfid_f3.1)) ++TEST ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_f3_shard1 ++ ++gfid_f4_shard1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/.shard/$gfid_f4.1)) ++TEST ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_f4_shard1 ++ ++#------------------------------------------------------------------------------- ++#gfid split-brain won't show up in split-brain count. ++EXPECT "3" afr_get_split_brain_count $V0 ++EXPECT_NOT "^0$" get_pending_heal_count $V0 ++ ++#Resolve split-brains ++TEST $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1 /file1 ++GFIDSTR="gfid:$gfid_f2_shard1" ++TEST $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1 $GFIDSTR ++TEST $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1 /.shard/$gfid_f3.1 ++TEST $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1 /.shard/$gfid_f4.1 ++TEST $CLI volume heal $V0 ++EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 ++cleanup; +diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c +index eef7fd2..32127c6 100644 +--- a/xlators/cluster/afr/src/afr-common.c ++++ b/xlators/cluster/afr/src/afr-common.c +@@ -2250,7 +2250,8 @@ afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this, + if ((spb_choice >= 0) && + (AFR_COUNT(success_replies, child_count) == child_count)) { + *read_subvol = spb_choice; +- } else if (!priv->quorum_count) { ++ } else if (!priv->quorum_count || ++ frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) { + *read_subvol = afr_first_up_child(frame, this); + } else if (priv->quorum_count && + afr_has_quorum(data_readable, this, NULL)) { +diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c +index 2e2ef5d..16d557b 100644 +--- a/xlators/features/shard/src/shard.c ++++ b/xlators/features/shard/src/shard.c +@@ -1472,7 +1472,8 @@ int shard_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, + shard_local_t *local = NULL; + + this->itable = loc->inode->table; +- if (frame->root->pid != GF_CLIENT_PID_GSYNCD) { ++ if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) && ++ (frame->root->pid != GF_CLIENT_PID_GLFS_HEAL)) { + SHARD_ENTRY_FOP_CHECK(loc, op_errno, err); + } + +-- +1.8.3.1 + |