ocfs2/cluster/quorum.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *
0004  * Copyright (C) 2005 Oracle.  All rights reserved.
0005  */
0006
0007 /* This quorum hack is only here until we transition to some more rational
0008  * approach that is driven from userspace.  Honest.  No foolin'.
0009  *
0010  * Imagine two nodes lose network connectivity to each other but they're still
0011  * up and operating in every other way.  Presumably a network timeout indicates
0012  * that a node is broken and should be recovered.  They can't both recover each
0013  * other and both carry on without serialising their access to the file system.
0014  * They need to decide who is authoritative.  Now extend that problem to
0015  * arbitrary groups of nodes losing connectivity between each other.
0016  *
0017  * So we declare that a node which has given up on connecting to a majority
0018  * of nodes who are still heartbeating will fence itself.
0019  *
0020  * There are huge opportunities for races here.  After we give up on a node's
0021  * connection we need to wait long enough to give heartbeat an opportunity
0022  * to declare the node as truly dead.  We also need to be careful with the
0023  * race between when we see a node start heartbeating and when we connect
0024  * to it.
0025  *
0026  * So nodes that are in this transtion put a hold on the quorum decision
0027  * with a counter.  As they fall out of this transition they drop the count
0028  * and if they're the last, they fire off the decision.
0029  */
0030 #include <linux/kernel.h>
0031 #include <linux/workqueue.h>
0032 #include <linux/reboot.h>
0033
0034 #include "heartbeat.h"
0035 #include "nodemanager.h"
0036 #define MLOG_MASK_PREFIX ML_QUORUM
0037 #include "masklog.h"
0038 #include "quorum.h"
0039
0040 static struct o2quo_state {
0041     spinlock_t      qs_lock;
0042     struct work_struct  qs_work;
0043     int         qs_pending;
0044     int         qs_heartbeating;
0045     unsigned long       qs_hb_bm[BITS_TO_LONGS(O2NM_MAX_NODES)];
0046     int         qs_connected;
0047     unsigned long       qs_conn_bm[BITS_TO_LONGS(O2NM_MAX_NODES)];
0048     int         qs_holds;
0049     unsigned long       qs_hold_bm[BITS_TO_LONGS(O2NM_MAX_NODES)];
0050 } o2quo_state;
0051
0052 /* this is horribly heavy-handed.  It should instead flip the file
0053  * system RO and call some userspace script. */
0054 static void o2quo_fence_self(void)
0055 {
0056     /* panic spins with interrupts enabled.  with preempt
0057      * threads can still schedule, etc, etc */
0058     o2hb_stop_all_regions();
0059
0060     switch (o2nm_single_cluster->cl_fence_method) {
0061     case O2NM_FENCE_PANIC:
0062         panic("*** ocfs2 is very sorry to be fencing this system by "
0063               "panicing ***\n");
0064         break;
0065     default:
0066         WARN_ON(o2nm_single_cluster->cl_fence_method >=
0067             O2NM_FENCE_METHODS);
0068         fallthrough;
0069     case O2NM_FENCE_RESET:
0070         printk(KERN_ERR "*** ocfs2 is very sorry to be fencing this "
0071                "system by restarting ***\n");
0072         emergency_restart();
0073         break;
0074     }
0075 }
0076
0077 /* Indicate that a timeout occurred on a heartbeat region write. The
0078  * other nodes in the cluster may consider us dead at that time so we
0079  * want to "fence" ourselves so that we don't scribble on the disk
0080  * after they think they've recovered us. This can't solve all
0081  * problems related to writeout after recovery but this hack can at
0082  * least close some of those gaps. When we have real fencing, this can
0083  * go away as our node would be fenced externally before other nodes
0084  * begin recovery. */
0085 void o2quo_disk_timeout(void)
0086 {
0087     o2quo_fence_self();
0088 }
0089
0090 static void o2quo_make_decision(struct work_struct *work)
0091 {
0092     int quorum;
0093     int lowest_hb, lowest_reachable = 0, fence = 0;
0094     struct o2quo_state *qs = &o2quo_state;
0095
0096     spin_lock(&qs->qs_lock);
0097
0098     lowest_hb = find_first_bit(qs->qs_hb_bm, O2NM_MAX_NODES);
0099     if (lowest_hb != O2NM_MAX_NODES)
0100         lowest_reachable = test_bit(lowest_hb, qs->qs_conn_bm);
0101
0102     mlog(0, "heartbeating: %d, connected: %d, "
0103          "lowest: %d (%sreachable)\n", qs->qs_heartbeating,
0104          qs->qs_connected, lowest_hb, lowest_reachable ? "" : "un");
0105
0106     if (!test_bit(o2nm_this_node(), qs->qs_hb_bm) ||
0107         qs->qs_heartbeating == 1)
0108         goto out;
0109
0110     if (qs->qs_heartbeating & 1) {
0111         /* the odd numbered cluster case is straight forward --
0112          * if we can't talk to the majority we're hosed */
0113         quorum = (qs->qs_heartbeating + 1)/2;
0114         if (qs->qs_connected < quorum) {
0115             mlog(ML_ERROR, "fencing this node because it is "
0116                  "only connected to %u nodes and %u is needed "
0117                  "to make a quorum out of %u heartbeating nodes\n",
0118                  qs->qs_connected, quorum,
0119                  qs->qs_heartbeating);
0120             fence = 1;
0121         }
0122     } else {
0123         /* the even numbered cluster adds the possibility of each half
0124          * of the cluster being able to talk amongst themselves.. in
0125          * that case we're hosed if we can't talk to the group that has
0126          * the lowest numbered node */
0127         quorum = qs->qs_heartbeating / 2;
0128         if (qs->qs_connected < quorum) {
0129             mlog(ML_ERROR, "fencing this node because it is "
0130                  "only connected to %u nodes and %u is needed "
0131                  "to make a quorum out of %u heartbeating nodes\n",
0132                  qs->qs_connected, quorum,
0133                  qs->qs_heartbeating);
0134             fence = 1;
0135         }
0136         else if ((qs->qs_connected == quorum) &&
0137              !lowest_reachable) {
0138             mlog(ML_ERROR, "fencing this node because it is "
0139                  "connected to a half-quorum of %u out of %u "
0140                  "nodes which doesn't include the lowest active "
0141                  "node %u\n", quorum, qs->qs_heartbeating,
0142                  lowest_hb);
0143             fence = 1;
0144         }
0145     }
0146
0147 out:
0148     if (fence) {
0149         spin_unlock(&qs->qs_lock);
0150         o2quo_fence_self();
0151     } else {
0152         mlog(ML_NOTICE, "not fencing this node, heartbeating: %d, "
0153             "connected: %d, lowest: %d (%sreachable)\n",
0154             qs->qs_heartbeating, qs->qs_connected, lowest_hb,
0155             lowest_reachable ? "" : "un");
0156         spin_unlock(&qs->qs_lock);
0157
0158     }
0159
0160 }
0161
0162 static void o2quo_set_hold(struct o2quo_state *qs, u8 node)
0163 {
0164     assert_spin_locked(&qs->qs_lock);
0165
0166     if (!test_and_set_bit(node, qs->qs_hold_bm)) {
0167         qs->qs_holds++;
0168         mlog_bug_on_msg(qs->qs_holds == O2NM_MAX_NODES,
0169                     "node %u\n", node);
0170         mlog(0, "node %u, %d total\n", node, qs->qs_holds);
0171     }
0172 }
0173
0174 static void o2quo_clear_hold(struct o2quo_state *qs, u8 node)
0175 {
0176     assert_spin_locked(&qs->qs_lock);
0177
0178     if (test_and_clear_bit(node, qs->qs_hold_bm)) {
0179         mlog(0, "node %u, %d total\n", node, qs->qs_holds - 1);
0180         if (--qs->qs_holds == 0) {
0181             if (qs->qs_pending) {
0182                 qs->qs_pending = 0;
0183                 schedule_work(&qs->qs_work);
0184             }
0185         }
0186         mlog_bug_on_msg(qs->qs_holds < 0, "node %u, holds %d\n",
0187                 node, qs->qs_holds);
0188     }
0189 }
0190
0191 /* as a node comes up we delay the quorum decision until we know the fate of
0192  * the connection.  the hold will be droped in conn_up or hb_down.  it might be
0193  * perpetuated by con_err until hb_down.  if we already have a conn, we might
0194  * be dropping a hold that conn_up got. */
0195 void o2quo_hb_up(u8 node)
0196 {
0197     struct o2quo_state *qs = &o2quo_state;
0198
0199     spin_lock(&qs->qs_lock);
0200
0201     qs->qs_heartbeating++;
0202     mlog_bug_on_msg(qs->qs_heartbeating == O2NM_MAX_NODES,
0203                 "node %u\n", node);
0204     mlog_bug_on_msg(test_bit(node, qs->qs_hb_bm), "node %u\n", node);
0205     set_bit(node, qs->qs_hb_bm);
0206
0207     mlog(0, "node %u, %d total\n", node, qs->qs_heartbeating);
0208
0209     if (!test_bit(node, qs->qs_conn_bm))
0210         o2quo_set_hold(qs, node);
0211     else
0212         o2quo_clear_hold(qs, node);
0213
0214     spin_unlock(&qs->qs_lock);
0215 }
0216
0217 /* hb going down releases any holds we might have had due to this node from
0218  * conn_up, conn_err, or hb_up */
0219 void o2quo_hb_down(u8 node)
0220 {
0221     struct o2quo_state *qs = &o2quo_state;
0222
0223     spin_lock(&qs->qs_lock);
0224
0225     qs->qs_heartbeating--;
0226     mlog_bug_on_msg(qs->qs_heartbeating < 0,
0227             "node %u, %d heartbeating\n",
0228             node, qs->qs_heartbeating);
0229     mlog_bug_on_msg(!test_bit(node, qs->qs_hb_bm), "node %u\n", node);
0230     clear_bit(node, qs->qs_hb_bm);
0231
0232     mlog(0, "node %u, %d total\n", node, qs->qs_heartbeating);
0233
0234     o2quo_clear_hold(qs, node);
0235
0236     spin_unlock(&qs->qs_lock);
0237 }
0238
0239 /* this tells us that we've decided that the node is still heartbeating
0240  * even though we've lost it's conn.  it must only be called after conn_err
0241  * and indicates that we must now make a quorum decision in the future,
0242  * though we might be doing so after waiting for holds to drain.  Here
0243  * we'll be dropping the hold from conn_err. */
0244 void o2quo_hb_still_up(u8 node)
0245 {
0246     struct o2quo_state *qs = &o2quo_state;
0247
0248     spin_lock(&qs->qs_lock);
0249
0250     mlog(0, "node %u\n", node);
0251
0252     qs->qs_pending = 1;
0253     o2quo_clear_hold(qs, node);
0254
0255     spin_unlock(&qs->qs_lock);
0256 }
0257
0258 /* This is analogous to hb_up.  as a node's connection comes up we delay the
0259  * quorum decision until we see it heartbeating.  the hold will be droped in
0260  * hb_up or hb_down.  it might be perpetuated by con_err until hb_down.  if
0261  * it's already heartbeating we might be dropping a hold that conn_up got.
0262  * */
0263 void o2quo_conn_up(u8 node)
0264 {
0265     struct o2quo_state *qs = &o2quo_state;
0266
0267     spin_lock(&qs->qs_lock);
0268
0269     qs->qs_connected++;
0270     mlog_bug_on_msg(qs->qs_connected == O2NM_MAX_NODES,
0271                 "node %u\n", node);
0272     mlog_bug_on_msg(test_bit(node, qs->qs_conn_bm), "node %u\n", node);
0273     set_bit(node, qs->qs_conn_bm);
0274
0275     mlog(0, "node %u, %d total\n", node, qs->qs_connected);
0276
0277     if (!test_bit(node, qs->qs_hb_bm))
0278         o2quo_set_hold(qs, node);
0279     else
0280         o2quo_clear_hold(qs, node);
0281
0282     spin_unlock(&qs->qs_lock);
0283 }
0284
0285 /* we've decided that we won't ever be connecting to the node again.  if it's
0286  * still heartbeating we grab a hold that will delay decisions until either the
0287  * node stops heartbeating from hb_down or the caller decides that the node is
0288  * still up and calls still_up */
0289 void o2quo_conn_err(u8 node)
0290 {
0291     struct o2quo_state *qs = &o2quo_state;
0292
0293     spin_lock(&qs->qs_lock);
0294
0295     if (test_bit(node, qs->qs_conn_bm)) {
0296         qs->qs_connected--;
0297         mlog_bug_on_msg(qs->qs_connected < 0,
0298                 "node %u, connected %d\n",
0299                 node, qs->qs_connected);
0300
0301         clear_bit(node, qs->qs_conn_bm);
0302
0303         if (test_bit(node, qs->qs_hb_bm))
0304             o2quo_set_hold(qs, node);
0305     }
0306
0307     mlog(0, "node %u, %d total\n", node, qs->qs_connected);
0308
0309
0310     spin_unlock(&qs->qs_lock);
0311 }
0312
0313 void o2quo_init(void)
0314 {
0315     struct o2quo_state *qs = &o2quo_state;
0316
0317     spin_lock_init(&qs->qs_lock);
0318     INIT_WORK(&qs->qs_work, o2quo_make_decision);
0319 }
0320
0321 void o2quo_exit(void)
0322 {
0323     struct o2quo_state *qs = &o2quo_state;
0324
0325     flush_work(&qs->qs_work);
0326 }