~ne02ptzero/libfloat

This thread contains a patchset. You're looking at the original emails, but you may wish to use the patch review UI. Review patch
1

[PATCH] Fix follower stuck after a snapshot

Michael Bonfils <mbonfils@scaleway.com>
Details
Message ID
<20220208135847.20743-1-mbonfils@scaleway.com>
DKIM signature
pass
Download raw message
Patch: +29 -33
Signed-off-by: Michael Bonfils <mbonfils@scaleway.com>
---
 election.c | 14 +++++++-------
 internal.h |  4 ++--
 libfloat.h |  2 +-
 log.c      | 29 ++++++++++++-----------------
 periodic.c |  5 +++--
 raft.c     |  8 ++++----
 6 files changed, 29 insertions(+), 33 deletions(-)

diff --git a/election.c b/election.c
index d9db208..3698149 100644
--- a/election.c
+++ b/election.c
@@ -18,13 +18,13 @@ static int libfloat_get_number_of_votes_for_me(libfloat_ctx_t *ctx)
    return ret;
}

void libfloat_election_start(libfloat_ctx_t *ctx)
void libfloat_election_start(libfloat_ctx_t *ctx, const char *reason)
{
    libfloat_node_t     *node;
    libfloat_term_t     last_term = 0;
    libfloat_entry_id_t last_id = 0;

    DEBUG(ctx, "Election starting!");
    DEBUG(ctx, "Election starting! reason=%s", reason);
    /* First, reset the vote of everyone */
    for_every_node(ctx, node, {
        node->has_voted_for_me = 0;
@@ -39,7 +39,7 @@ void libfloat_election_start(libfloat_ctx_t *ctx)

    /* Discard the current leader and become candidate */
    ctx->leader = NULL;
    libfloat_become_candidate(ctx);
    libfloat_become_candidate(ctx, "new election");

    /* Randomize election timeout, and reset counter */
    ctx->election_timeout_rand = ctx->conf.election_timeout + ctx->rand() % ctx->conf.election_timeout;
@@ -101,7 +101,7 @@ static bool libfloat_can_i_grant_vote(libfloat_ctx_t *ctx, libfloat_rpc_request_
        /* We have a superior log, we can't grant our vote */

        /* There's an election in progress, and we have a superior log, let's try to become leader */
        libfloat_election_start(ctx);
        libfloat_election_start(ctx, "I have a greater log");
        return false;
    }

@@ -131,7 +131,7 @@ void libfloat_request_vote_receive(libfloat_ctx_t *ctx, libfloat_rpc_request_vot
    {
        libfloat_set_current_term(ctx, req->term);

        libfloat_become_follower(ctx);
        libfloat_become_follower(ctx, "request_vote_receive: our term is lower than remote term");
        ctx->timeout_elapsed = 0;
    }

@@ -174,7 +174,7 @@ void libfloat_request_vote_response(libfloat_ctx_t *ctx, libfloat_rpc_response_v
    {
        libfloat_set_current_term(ctx, resp->term);

        libfloat_become_follower(ctx);
        libfloat_become_follower(ctx, "request_vote_response: our term is lower than remote");
        ctx->leader = NULL;
        return;
    }
@@ -230,5 +230,5 @@ void libfloat_election_resign(libfloat_ctx_t *ctx)
void libfloat_election_resign_receive(libfloat_ctx_t *ctx)
{
    ERROR(ctx, "Restart election");
    libfloat_election_start(ctx);
    libfloat_election_start(ctx, "Resign received");
}
diff --git a/internal.h b/internal.h
index 0f59c30..a65ed23 100644
--- a/internal.h
+++ b/internal.h
@@ -17,7 +17,7 @@
 *
 * \param[in] ctx libfloat context
 */
void libfloat_election_start(libfloat_ctx_t *ctx);
void libfloat_election_start(libfloat_ctx_t *ctx, const char *reason);

/*!
 * \brief Set term to persistent storage
@@ -48,7 +48,7 @@ void libfloat_set_current_commit_index(libfloat_ctx_t *ctx, libfloat_entry_id_t
 *
 * \param[in] ctx libfloat context
 */
void libfloat_become_candidate(libfloat_ctx_t *ctx);
void libfloat_become_candidate(libfloat_ctx_t *ctx, const char *reason);

/*!
 * \brief Send AE request to a node
diff --git a/libfloat.h b/libfloat.h
index d2cf009..c7a8df4 100644
--- a/libfloat.h
+++ b/libfloat.h
@@ -442,7 +442,7 @@ void libfloat_become_leader(libfloat_ctx_t *ctx);
 *
 * \param[in] ctx libfloat context
 */
void libfloat_become_follower(libfloat_ctx_t *ctx);
void libfloat_become_follower(libfloat_ctx_t *ctx, const char *reason);

/*!
 * \brief Leave cluster
diff --git a/log.c b/log.c
index 03c043c..e31496e 100644
--- a/log.c
+++ b/log.c
@@ -257,12 +257,7 @@ void libfloat_send_append_entries(libfloat_ctx_t *ctx, libfloat_node_t *node, bo
        goto end;
    }

    if (node->next_log_to_send == ctx->persistent.snapshot.index)
    {
        /* Snapshot log was sent earlier, no-op till the node replies */
        return;
    }
    else if (node->next_log_to_send < ctx->persistent.snapshot.index)
    if (node->next_log_to_send <= ctx->persistent.snapshot.index)
    {
        if (node->snapshot_count > 5)
        {
@@ -300,7 +295,7 @@ void libfloat_send_append_entries(libfloat_ctx_t *ctx, libfloat_node_t *node, bo
                {
                    /* Log is also not in our database, that's very bad, let's step down from leadership */
                    ERROR(ctx, "libfloat_send_append_entries: Can't find log %u in our hashtable, stepping down from leadership", j);
                    libfloat_become_follower(ctx);
                    libfloat_become_follower(ctx, "Can't find log, stepping down from leadership");
                    ctx->stepping_down = true;
                    return;
                }
@@ -349,7 +344,7 @@ void libfloat_send_append_entries(libfloat_ctx_t *ctx, libfloat_node_t *node, bo
            {
                /* Can't find that log, that's probably very bad, let's step down from leadership */
                ERROR(ctx, "libfloat_send_append_entries: Can't find the previous log to send node->next_log_to_send=%u id=%u, stepping down from leadership", node->next_log_to_send, node_prev_id);
                libfloat_become_follower(ctx);
                libfloat_become_follower(ctx, "Failed to find previous log to send");
                ctx->stepping_down = true;
                return;
            }
@@ -386,12 +381,12 @@ void libfloat_append_entries_receive(libfloat_ctx_t *ctx, libfloat_rpc_append_en
    if (ctx->state == RAFT_STATE_CANDIDATE && ctx->persistent.term == req->term)
    {
        /* Someone has been elected! */
        libfloat_become_follower(ctx);
        libfloat_become_follower(ctx, "Someone has been elected");
    }
    else if (req->term > ctx->persistent.term)
    {
        libfloat_set_current_term(ctx, req->term);
        libfloat_become_follower(ctx);
        libfloat_become_follower(ctx, "Remote term is greater than our term");
    }
    else if (req->term < ctx->persistent.term)
    {
@@ -399,13 +394,13 @@ void libfloat_append_entries_receive(libfloat_ctx_t *ctx, libfloat_rpc_append_en
        {
            /* Leader has a term lower than us but more log, let's rollback our term. */
            libfloat_set_current_term(ctx, req->term);
            libfloat_become_follower(ctx);
            libfloat_become_follower(ctx, "leader has a lower term but more log");
        }
        else
        {
            /* Who are you and how are you in a position of leadership? */
            DEBUG(ctx, "libfloat_append_entries_receive: Dropping AE term=%d remote_term=%d", ctx->persistent.term, req->term);
            libfloat_election_start(ctx);
            libfloat_election_start(ctx, "Dropping mismatch AE term");
            return;
        }
    }
@@ -564,7 +559,7 @@ response:
        }

        resp->current_index = log->id;
     }
    }

end:
    if (!resp->success)
@@ -617,8 +612,8 @@ void libfloat_append_entries_response(libfloat_ctx_t *ctx, libfloat_rpc_append_e

    if (resp->current_index != resp->first_index)
    {
        DEBUG(ctx, "Received AE response from node %d: term=%d success=%d current_index=%d first_index=%d",
            node->id, resp->term, resp->success, resp->current_index, resp->first_index
        DEBUG(ctx, "Received AE response from node %d (hearbeating=%d): term=%d success=%d current_index=%d first_index=%d",
            node->id, node->hearbeating, resp->term, resp->success, resp->current_index, resp->first_index
        );
    }

@@ -630,7 +625,7 @@ void libfloat_append_entries_response(libfloat_ctx_t *ctx, libfloat_rpc_append_e
        node->replicated_log = resp->current_index;
    }

    if (node->next_log_to_send <= resp->current_index && node->hearbeating)
    if (node->next_log_to_send <= resp->current_index && (node->hearbeating || resp->current_index == ctx->persistent.snapshot.index))
    {
        node->next_log_to_send = resp->current_index + 1;
    }
@@ -643,7 +638,7 @@ void libfloat_append_entries_response(libfloat_ctx_t *ctx, libfloat_rpc_append_e
    if (ctx->persistent.term < resp->term)
    {
        libfloat_set_current_term(ctx, resp->term);
        libfloat_become_follower(ctx);
        libfloat_become_follower(ctx, "our term is lower than remote term");
        ctx->leader = NULL;
    }
    else if (ctx->persistent.term != resp->term)
diff --git a/periodic.c b/periodic.c
index 6e9b0ec..ce9a31f 100644
--- a/periodic.c
+++ b/periodic.c
@@ -30,7 +30,7 @@ void libfloat_periodic(libfloat_ctx_t *ctx, uint32_t time)
                    * Hmmm, we might be partitionned, or at the very least we don't have a quorum anymore.
                    * Time to step down!
                    */
                    libfloat_become_follower(ctx);
                    libfloat_become_follower(ctx, "not enought nodes reachable");
                    ERROR(ctx, "Sanity timeout has been reached (%d seconds), stepping down from leader position: reachable %d / quorum %lu (%lu)",
                          ctx->conf.sanity_timeout, nodes_reachable, ctx->n_nodes / 2 + 1, ctx->n_nodes);
                    return;
@@ -89,7 +89,8 @@ void libfloat_periodic(libfloat_ctx_t *ctx, uint32_t time)
        if (!ctx->is_snapshotting && !ctx->stepping_down)
        {
            /* We don't want leadership to change in the middle of a snapshot */
            libfloat_election_start(ctx);
            ERROR(ctx, "New election: timeout elapsed %d", ctx->timeout_elapsed);
            libfloat_election_start(ctx, "Election timeout, start a new one");
        }
    }
}
diff --git a/raft.c b/raft.c
index bde3522..e73e516 100644
--- a/raft.c
+++ b/raft.c
@@ -25,16 +25,16 @@ void libfloat_become_leader(libfloat_ctx_t *ctx)
    ctx->me->is_up_to_date = 1;
}

void libfloat_become_candidate(libfloat_ctx_t *ctx)
void libfloat_become_candidate(libfloat_ctx_t *ctx, const char *reason)
{
    DEBUG(ctx, "Becoming candidate");
    DEBUG(ctx, "Becoming candidate: reason=%s", reason);
    ctx->state = RAFT_STATE_CANDIDATE;
    ctx->election_timeout_rand = ctx->conf.election_timeout + ctx->rand() % ctx->conf.election_timeout;
}

void libfloat_become_follower(libfloat_ctx_t *ctx)
void libfloat_become_follower(libfloat_ctx_t *ctx, const char *reason)
{
    DEBUG(ctx, "Becoming follower");
    DEBUG(ctx, "Becoming follower: reason=%s", reason);
    ctx->abort_send_snapshot(ctx);
    ctx->state = RAFT_STATE_FOLLOWER;
    ctx->election_timeout_rand = ctx->conf.election_timeout + ctx->rand() % ctx->conf.election_timeout;
-- 
2.25.1
Details
Message ID
<20220208144114.6puryyjoxfuklvqe@scaleway>
In-Reply-To
<20220208135847.20743-1-mbonfils@scaleway.com> (view parent)
DKIM signature
pass
Download raw message
LG
Reply to thread Export thread (mbox)