Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fw_update: implement retry with backoff #703

Merged
merged 4 commits into from
Dec 13, 2024
Merged

fw_update: implement retry with backoff #703

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3b8ba78
fw_update: add variables and functions to track retries with backoff
szczys Dec 9, 2024
bee5c5e
fw_update: add backoff awareness to received_new_target_component()
szczys Dec 9, 2024
9c6bd4d
fw_download: implement retry with backoff for firmware update
szczys Dec 9, 2024
f18f360
fw_update: when reporting IDLE during retry, use await retry reason
szczys Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 115 additions & 13 deletions src/fw_update.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
{
struct golioth_fw_update_config config;
struct golioth_ota_component target_component;
uint32_t backoff_duration_ms;
uint32_t last_fail_ts;
};

struct download_progress_context
Expand All @@ -42,6 +44,8 @@
#define FW_REPORT_BACKOFF_MAX_S 180
#define FW_REPORT_MAX_RETRIES 5
#define FW_REPORT_RETRIES_INITAL_DELAY_S 5
#define BACKOFF_DURATION_INITIAL_MS 60 * 1000
#define BACKOFF_DURATION_MAX_MS 24 * 60 * 60 * 1000

#define FW_REPORT_COMPONENT_NAME 1 << 0
#define FW_REPORT_TARGET_VERSION 1 << 1
Expand Down Expand Up @@ -164,9 +168,50 @@
golioth_sys_sem_give(_manifest_rcvd);
}

static void backoff_reset(struct fw_update_component_context *ctx)

Check warning on line 171 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L171 

Added line #L171 was not covered by tests
{
ctx->backoff_duration_ms = 0;
ctx->last_fail_ts = 0;
}

Check warning on line 175 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L173-L175 

Added lines #L173 - L175 were not covered by tests

static void backoff_increment(struct fw_update_component_context *ctx)

Check warning on line 177 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L177 

Added line #L177 was not covered by tests
{
if (ctx->backoff_duration_ms == 0)
{
ctx->backoff_duration_ms = BACKOFF_DURATION_INITIAL_MS;

Check warning on line 181 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L181 

Added line #L181 was not covered by tests
}
else
{
ctx->backoff_duration_ms = ctx->backoff_duration_ms * 2;

Check warning on line 185 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L185 

Added line #L185 was not covered by tests

if (ctx->backoff_duration_ms > BACKOFF_DURATION_MAX_MS)
{
ctx->backoff_duration_ms = BACKOFF_DURATION_MAX_MS;

Check warning on line 189 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L189 

Added line #L189 was not covered by tests
}
}

ctx->last_fail_ts = (uint32_t) golioth_sys_now_ms();
}

Check warning on line 194 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L193-L194 

Added lines #L193 - L194 were not covered by tests

static int32_t backoff_ms_before_expiration(struct fw_update_component_context *ctx)

Check warning on line 196 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L196 

Added line #L196 was not covered by tests
{
uint32_t actual_duration = ((uint32_t) golioth_sys_now_ms()) - ctx->last_fail_ts;

Check warning on line 198 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L198 

Added line #L198 was not covered by tests

if (actual_duration < ctx->backoff_duration_ms)
{
return (int32_t) (ctx->backoff_duration_ms - actual_duration);

Check warning on line 202 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L202 

Added line #L202 was not covered by tests
}

return 0;

Check warning on line 205 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L205 

Added line #L205 was not covered by tests
}

static bool received_new_target_component(const struct golioth_ota_manifest *manifest,
struct fw_update_component_context *ctx)
{
golioth_sys_mutex_lock(_manifest_update_mut, GOLIOTH_SYS_WAIT_FOREVER);

Check warning on line 211 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L211 

Added line #L211 was not covered by tests

bool found_new = false;

Check warning on line 213 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L213 

Added line #L213 was not covered by tests

const struct golioth_ota_component *new_component =
golioth_ota_find_component(manifest, ctx->config.fw_package_name);
if (new_component)
Expand All @@ -176,10 +221,20 @@
ctx->config.current_version,
new_component->version);

if (0 != strcmp(ctx->config.current_version, new_component->version))
if (0 == strcmp(ctx->config.current_version, new_component->version))
{
GLTH_LOGI(TAG, "Current version matches target version.");
}
else if (ctx->backoff_duration_ms
&& 0 == strcmp(ctx->target_component.version, new_component->version))
{
GLTH_LOGI(TAG, "Update to target version already in progress.");
}
else
{
memcpy(&ctx->target_component, new_component, sizeof(struct golioth_ota_component));
return true;
backoff_reset(ctx);
found_new = true;

Check warning on line 237 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L236-L237 

Added lines #L236 - L237 were not covered by tests
}
}
else
Expand All @@ -193,7 +248,9 @@
*/
}

return false;
golioth_sys_mutex_unlock(_manifest_update_mut);

Check warning on line 251 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L251 

Added line #L251 was not covered by tests

return found_new;

Check warning on line 253 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L253 

Added line #L253 was not covered by tests
}

static void fw_observe_manifest(void)
Expand Down Expand Up @@ -320,18 +377,54 @@
| FW_REPORT_TARGET_VERSION);

GLTH_LOGI(TAG, "Waiting to receive OTA manifest");
golioth_sys_sem_take(_manifest_rcvd, GOLIOTH_SYS_WAIT_FOREVER);
GLTH_LOGI(TAG, "Received OTA manifest");

golioth_sys_mutex_lock(_manifest_update_mut, GOLIOTH_SYS_WAIT_FOREVER);
bool new_component_received =
received_new_target_component(&_ota_manifest, &_component_ctx);
golioth_sys_mutex_unlock(_manifest_update_mut);

if (!new_component_received)
while (1)
{
GLTH_LOGI(TAG, "Manifest does not contain different firmware version. Nothing to do.");
continue;
int32_t manifest_timeout = (_component_ctx.backoff_duration_ms == 0)

Check warning on line 383 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L383 

Added line #L383 was not covered by tests
? GOLIOTH_SYS_WAIT_FOREVER
: backoff_ms_before_expiration(&_component_ctx);

GLTH_LOGI(TAG, "State = Idle");

if (manifest_timeout == GOLIOTH_SYS_WAIT_FOREVER)
{
golioth_fw_update_report_state_sync(&_component_ctx,

Check warning on line 391 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L391 

Added line #L391 was not covered by tests
GOLIOTH_OTA_STATE_IDLE,
GOLIOTH_OTA_REASON_READY,
FW_REPORT_COMPONENT_NAME
| FW_REPORT_CURRENT_VERSION);
}
else
{
golioth_fw_update_report_state_sync(&_component_ctx,

Check warning on line 399 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L399 

Added line #L399 was not covered by tests
GOLIOTH_OTA_STATE_IDLE,
GOLIOTH_OTA_REASON_AWAIT_RETRY,
FW_REPORT_COMPONENT_NAME
| FW_REPORT_CURRENT_VERSION
| FW_REPORT_TARGET_VERSION);
}

if (!golioth_sys_sem_take(_manifest_rcvd, manifest_timeout))
{
GLTH_LOGI(TAG,
"Retry component download: %s",
_component_ctx.config.fw_package_name);
break;

Check warning on line 412 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L412 

Added line #L412 was not covered by tests
}

GLTH_LOGI(TAG, "Received OTA manifest");

bool new_component_received =
received_new_target_component(&_ota_manifest, &_component_ctx);

Check warning on line 418 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L418 

Added line #L418 was not covered by tests

if (!new_component_received)
{
GLTH_LOGI(TAG,
"Manifest does not contain different firmware version. Nothing to do.");
continue;

Check warning on line 424 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L424 

Added line #L424 was not covered by tests
}

break;

Check warning on line 427 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L427 

Added line #L427 was not covered by tests
}

GLTH_LOGI(TAG, "State = Downloading");
Expand Down Expand Up @@ -427,6 +520,9 @@
}
}

/* Download finished, prepare backoff in case needed */
backoff_increment(&_component_ctx);

Check warning on line 524 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L524 

Added line #L524 was not covered by tests

if (err != GOLIOTH_OK)
{
switch (err)
Expand Down Expand Up @@ -497,6 +593,9 @@
continue;
}

/* Download successful. Reset backoff */
backoff_reset(&_component_ctx);

Check warning on line 597 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L597 

Added line #L597 was not covered by tests

int countdown = 5;
while (countdown > 0)
{
Expand All @@ -523,7 +622,10 @@
static bool initialized = false;

_client = client;

_component_ctx.config = *config;
backoff_reset(&_component_ctx);

Check warning on line 627 in src/fw_update.c

View check run for this annotation

Codecov / codecov/patch

src/fw_update.c#L627 

Added line #L627 was not covered by tests

_manifest_update_mut = golioth_sys_mutex_create(); // never destroyed
_manifest_rcvd = golioth_sys_sem_create(1, 0); // never destroyed

Expand Down
Loading