Michael Bonfils: 1 Fix follower stuck after a snapshot 6 files changed, 29 insertions(+), 33 deletions(-)
Copy & paste the following snippet into your terminal to import this patchset into git:
curl -s https://lists.sr.ht/~ne02ptzero/libfloat/patches/29194/mbox | git am -3Learn more about email & git
Signed-off-by: Michael Bonfils <mbonfils@scaleway.com> --- election.c | 14 +++++++------- internal.h | 4 ++-- libfloat.h | 2 +- log.c | 29 ++++++++++++----------------- periodic.c | 5 +++-- raft.c | 8 ++++---- 6 files changed, 29 insertions(+), 33 deletions(-) diff --git a/election.c b/election.c index d9db208..3698149 100644 --- a/election.c +++ b/election.c @@ -18,13 +18,13 @@ static int libfloat_get_number_of_votes_for_me(libfloat_ctx_t *ctx) return ret; } -void libfloat_election_start(libfloat_ctx_t *ctx) +void libfloat_election_start(libfloat_ctx_t *ctx, const char *reason) { libfloat_node_t *node; libfloat_term_t last_term = 0; libfloat_entry_id_t last_id = 0; - DEBUG(ctx, "Election starting!"); + DEBUG(ctx, "Election starting! reason=%s", reason); /* First, reset the vote of everyone */ for_every_node(ctx, node, { node->has_voted_for_me = 0; @@ -39,7 +39,7 @@ void libfloat_election_start(libfloat_ctx_t *ctx) /* Discard the current leader and become candidate */ ctx->leader = NULL; - libfloat_become_candidate(ctx); + libfloat_become_candidate(ctx, "new election"); /* Randomize election timeout, and reset counter */ ctx->election_timeout_rand = ctx->conf.election_timeout + ctx->rand() % ctx->conf.election_timeout; @@ -101,7 +101,7 @@ static bool libfloat_can_i_grant_vote(libfloat_ctx_t *ctx, libfloat_rpc_request_ /* We have a superior log, we can't grant our vote */ /* There's an election in progress, and we have a superior log, let's try to become leader */ - libfloat_election_start(ctx); + libfloat_election_start(ctx, "I have a greater log"); return false; } @@ -131,7 +131,7 @@ void libfloat_request_vote_receive(libfloat_ctx_t *ctx, libfloat_rpc_request_vot { libfloat_set_current_term(ctx, req->term); - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "request_vote_receive: our term is lower than remote term"); ctx->timeout_elapsed = 0; } @@ -174,7 +174,7 @@ void libfloat_request_vote_response(libfloat_ctx_t *ctx, libfloat_rpc_response_v { libfloat_set_current_term(ctx, resp->term); - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "request_vote_response: our term is lower than remote"); ctx->leader = NULL; return; } @@ -230,5 +230,5 @@ void libfloat_election_resign(libfloat_ctx_t *ctx) void libfloat_election_resign_receive(libfloat_ctx_t *ctx) { ERROR(ctx, "Restart election"); - libfloat_election_start(ctx); + libfloat_election_start(ctx, "Resign received"); } diff --git a/internal.h b/internal.h index 0f59c30..a65ed23 100644 --- a/internal.h +++ b/internal.h @@ -17,7 +17,7 @@ * * \param[in] ctx libfloat context */ -void libfloat_election_start(libfloat_ctx_t *ctx); +void libfloat_election_start(libfloat_ctx_t *ctx, const char *reason); /*! * \brief Set term to persistent storage @@ -48,7 +48,7 @@ void libfloat_set_current_commit_index(libfloat_ctx_t *ctx, libfloat_entry_id_t * * \param[in] ctx libfloat context */ -void libfloat_become_candidate(libfloat_ctx_t *ctx); +void libfloat_become_candidate(libfloat_ctx_t *ctx, const char *reason); /*! * \brief Send AE request to a node diff --git a/libfloat.h b/libfloat.h index d2cf009..c7a8df4 100644 --- a/libfloat.h +++ b/libfloat.h @@ -442,7 +442,7 @@ void libfloat_become_leader(libfloat_ctx_t *ctx); * * \param[in] ctx libfloat context */ -void libfloat_become_follower(libfloat_ctx_t *ctx); +void libfloat_become_follower(libfloat_ctx_t *ctx, const char *reason); /*! * \brief Leave cluster diff --git a/log.c b/log.c index 03c043c..e31496e 100644 --- a/log.c +++ b/log.c @@ -257,12 +257,7 @@ void libfloat_send_append_entries(libfloat_ctx_t *ctx, libfloat_node_t *node, bo goto end; } - if (node->next_log_to_send == ctx->persistent.snapshot.index) - { - /* Snapshot log was sent earlier, no-op till the node replies */ - return; - } - else if (node->next_log_to_send < ctx->persistent.snapshot.index) + if (node->next_log_to_send <= ctx->persistent.snapshot.index) { if (node->snapshot_count > 5) { @@ -300,7 +295,7 @@ void libfloat_send_append_entries(libfloat_ctx_t *ctx, libfloat_node_t *node, bo { /* Log is also not in our database, that's very bad, let's step down from leadership */ ERROR(ctx, "libfloat_send_append_entries: Can't find log %u in our hashtable, stepping down from leadership", j); - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "Can't find log, stepping down from leadership"); ctx->stepping_down = true; return; } @@ -349,7 +344,7 @@ void libfloat_send_append_entries(libfloat_ctx_t *ctx, libfloat_node_t *node, bo { /* Can't find that log, that's probably very bad, let's step down from leadership */ ERROR(ctx, "libfloat_send_append_entries: Can't find the previous log to send node->next_log_to_send=%u id=%u, stepping down from leadership", node->next_log_to_send, node_prev_id); - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "Failed to find previous log to send"); ctx->stepping_down = true; return; } @@ -386,12 +381,12 @@ void libfloat_append_entries_receive(libfloat_ctx_t *ctx, libfloat_rpc_append_en if (ctx->state == RAFT_STATE_CANDIDATE && ctx->persistent.term == req->term) { /* Someone has been elected! */ - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "Someone has been elected"); } else if (req->term > ctx->persistent.term) { libfloat_set_current_term(ctx, req->term); - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "Remote term is greater than our term"); } else if (req->term < ctx->persistent.term) { @@ -399,13 +394,13 @@ void libfloat_append_entries_receive(libfloat_ctx_t *ctx, libfloat_rpc_append_en { /* Leader has a term lower than us but more log, let's rollback our term. */ libfloat_set_current_term(ctx, req->term); - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "leader has a lower term but more log"); } else { /* Who are you and how are you in a position of leadership? */ DEBUG(ctx, "libfloat_append_entries_receive: Dropping AE term=%d remote_term=%d", ctx->persistent.term, req->term); - libfloat_election_start(ctx); + libfloat_election_start(ctx, "Dropping mismatch AE term"); return; } } @@ -564,7 +559,7 @@ response: } resp->current_index = log->id; - } + } end: if (!resp->success) @@ -617,8 +612,8 @@ void libfloat_append_entries_response(libfloat_ctx_t *ctx, libfloat_rpc_append_e if (resp->current_index != resp->first_index) { - DEBUG(ctx, "Received AE response from node %d: term=%d success=%d current_index=%d first_index=%d", - node->id, resp->term, resp->success, resp->current_index, resp->first_index + DEBUG(ctx, "Received AE response from node %d (hearbeating=%d): term=%d success=%d current_index=%d first_index=%d", + node->id, node->hearbeating, resp->term, resp->success, resp->current_index, resp->first_index ); } @@ -630,7 +625,7 @@ void libfloat_append_entries_response(libfloat_ctx_t *ctx, libfloat_rpc_append_e node->replicated_log = resp->current_index; } - if (node->next_log_to_send <= resp->current_index && node->hearbeating) + if (node->next_log_to_send <= resp->current_index && (node->hearbeating || resp->current_index == ctx->persistent.snapshot.index)) { node->next_log_to_send = resp->current_index + 1; } @@ -643,7 +638,7 @@ void libfloat_append_entries_response(libfloat_ctx_t *ctx, libfloat_rpc_append_e if (ctx->persistent.term < resp->term) { libfloat_set_current_term(ctx, resp->term); - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "our term is lower than remote term"); ctx->leader = NULL; } else if (ctx->persistent.term != resp->term) diff --git a/periodic.c b/periodic.c index 6e9b0ec..ce9a31f 100644 --- a/periodic.c +++ b/periodic.c @@ -30,7 +30,7 @@ void libfloat_periodic(libfloat_ctx_t *ctx, uint32_t time) * Hmmm, we might be partitionned, or at the very least we don't have a quorum anymore. * Time to step down! */ - libfloat_become_follower(ctx); + libfloat_become_follower(ctx, "not enought nodes reachable"); ERROR(ctx, "Sanity timeout has been reached (%d seconds), stepping down from leader position: reachable %d / quorum %lu (%lu)", ctx->conf.sanity_timeout, nodes_reachable, ctx->n_nodes / 2 + 1, ctx->n_nodes); return; @@ -89,7 +89,8 @@ void libfloat_periodic(libfloat_ctx_t *ctx, uint32_t time) if (!ctx->is_snapshotting && !ctx->stepping_down) { /* We don't want leadership to change in the middle of a snapshot */ - libfloat_election_start(ctx); + ERROR(ctx, "New election: timeout elapsed %d", ctx->timeout_elapsed); + libfloat_election_start(ctx, "Election timeout, start a new one"); } } } diff --git a/raft.c b/raft.c index bde3522..e73e516 100644 --- a/raft.c +++ b/raft.c @@ -25,16 +25,16 @@ void libfloat_become_leader(libfloat_ctx_t *ctx) ctx->me->is_up_to_date = 1; } -void libfloat_become_candidate(libfloat_ctx_t *ctx) +void libfloat_become_candidate(libfloat_ctx_t *ctx, const char *reason) { - DEBUG(ctx, "Becoming candidate"); + DEBUG(ctx, "Becoming candidate: reason=%s", reason); ctx->state = RAFT_STATE_CANDIDATE; ctx->election_timeout_rand = ctx->conf.election_timeout + ctx->rand() % ctx->conf.election_timeout; } -void libfloat_become_follower(libfloat_ctx_t *ctx) +void libfloat_become_follower(libfloat_ctx_t *ctx, const char *reason) { - DEBUG(ctx, "Becoming follower"); + DEBUG(ctx, "Becoming follower: reason=%s", reason); ctx->abort_send_snapshot(ctx); ctx->state = RAFT_STATE_FOLLOWER; ctx->election_timeout_rand = ctx->conf.election_timeout + ctx->rand() % ctx->conf.election_timeout; -- 2.25.1
LG