From 3aa1975c1db1abd9bc1a75c4dcfbb172fcf5ba10 Mon Sep 17 00:00:00 2001 From: Andrew Gregory Date: Wed, 22 Nov 2023 14:19:21 -0800 Subject: [PATCH] alpm: add cache server support Cache servers differ from regular servers in that they do not produce warnings and are not removed from the server pool for "soft errors" (i.e. the server was reachable, but the download failed) and they are not used for databases. If a host is used for both a cache server and a regular server, it may still be removed from the server pool for soft errors that occur when used as cache server and removal from the server pool for soft errors will not affect future attempted use as a cache server. Signed-off-by: Andrew Gregory --- lib/libalpm/alpm.h | 28 ++++++++++++++ lib/libalpm/db.c | 69 +++++++++++++++++++++++++++++++++ lib/libalpm/db.h | 1 + lib/libalpm/dload.c | 93 ++++++++++++++++++++++++++++++++------------- lib/libalpm/dload.h | 2 + lib/libalpm/sync.c | 1 + 6 files changed, 167 insertions(+), 27 deletions(-) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index f372355c..d8439050 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -1343,6 +1343,34 @@ int alpm_db_add_server(alpm_db_t *db, const char *url); */ int alpm_db_remove_server(alpm_db_t *db, const char *url); +/** Get the list of cache servers assigned to this db. + * @param db pointer to the database to get the servers from + * @return a char* list of servers + */ +alpm_list_t *alpm_db_get_cache_servers(const alpm_db_t *db); + +/** Sets the list of cache servers for the database to use. + * @param db the database to set the servers. The list will be duped and + * the original will still need to be freed by the caller. + * @param servers a char* list of servers. + */ +int alpm_db_set_cache_servers(alpm_db_t *db, alpm_list_t *servers); + +/** Add a download cache server to a database. + * @param db database pointer + * @param url url of the server + * @return 0 on success, -1 on error (pm_errno is set accordingly) + */ +int alpm_db_add_cache_server(alpm_db_t *db, const char *url); + +/** Remove a download cache server from a database. + * @param db database pointer + * @param url url of the server + * @return 0 on success, 1 on server not present, + * -1 on error (pm_errno is set accordingly) + */ +int alpm_db_remove_cache_server(alpm_db_t *db, const char *url); + /* End of server accessors */ /** @} */ diff --git a/lib/libalpm/db.c b/lib/libalpm/db.c index ece8eae6..14f9e2e9 100644 --- a/lib/libalpm/db.c +++ b/lib/libalpm/db.c @@ -131,6 +131,26 @@ int SYMEXPORT alpm_db_unregister(alpm_db_t *db) return 0; } +alpm_list_t SYMEXPORT *alpm_db_get_cache_servers(const alpm_db_t *db) +{ + ASSERT(db != NULL, return NULL); + return db->cache_servers; +} + +int SYMEXPORT alpm_db_set_cache_servers(alpm_db_t *db, alpm_list_t *cache_servers) +{ + alpm_list_t *i; + ASSERT(db != NULL, return -1); + FREELIST(db->cache_servers); + for(i = cache_servers; i; i = i->next) { + char *url = i->data; + if(alpm_db_add_cache_server(db, url) != 0) { + return -1; + } + } + return 0; +} + alpm_list_t SYMEXPORT *alpm_db_get_servers(const alpm_db_t *db) { ASSERT(db != NULL, return NULL); @@ -164,6 +184,26 @@ static char *sanitize_url(const char *url) return newurl; } +int SYMEXPORT alpm_db_add_cache_server(alpm_db_t *db, const char *url) +{ + char *newurl; + + /* Sanity checks */ + ASSERT(db != NULL, return -1); + db->handle->pm_errno = ALPM_ERR_OK; + ASSERT(url != NULL && strlen(url) != 0, RET_ERR(db->handle, ALPM_ERR_WRONG_ARGS, -1)); + + newurl = sanitize_url(url); + if(!newurl) { + return -1; + } + db->cache_servers = alpm_list_add(db->cache_servers, newurl); + _alpm_log(db->handle, ALPM_LOG_DEBUG, "adding new cache server URL to database '%s': %s\n", + db->treename, newurl); + + return 0; +} + int SYMEXPORT alpm_db_add_server(alpm_db_t *db, const char *url) { char *newurl; @@ -184,6 +224,34 @@ int SYMEXPORT alpm_db_add_server(alpm_db_t *db, const char *url) return 0; } +int SYMEXPORT alpm_db_remove_cache_server(alpm_db_t *db, const char *url) +{ + char *newurl, *vdata = NULL; + int ret = 1; + + /* Sanity checks */ + ASSERT(db != NULL, return -1); + db->handle->pm_errno = ALPM_ERR_OK; + ASSERT(url != NULL && strlen(url) != 0, RET_ERR(db->handle, ALPM_ERR_WRONG_ARGS, -1)); + + newurl = sanitize_url(url); + if(!newurl) { + return -1; + } + + db->cache_servers = alpm_list_remove_str(db->cache_servers, newurl, &vdata); + + if(vdata) { + _alpm_log(db->handle, ALPM_LOG_DEBUG, "removed cache server URL from database '%s': %s\n", + db->treename, newurl); + free(vdata); + ret = 0; + } + + free(newurl); + return ret; +} + int SYMEXPORT alpm_db_remove_server(alpm_db_t *db, const char *url) { char *newurl, *vdata = NULL; @@ -328,6 +396,7 @@ void _alpm_db_free(alpm_db_t *db) /* cleanup pkgcache */ _alpm_db_free_pkgcache(db); /* cleanup server list */ + FREELIST(db->cache_servers); FREELIST(db->servers); FREE(db->_path); FREE(db->treename); diff --git a/lib/libalpm/db.h b/lib/libalpm/db.h index c9400365..1d22c9c1 100644 --- a/lib/libalpm/db.h +++ b/lib/libalpm/db.h @@ -69,6 +69,7 @@ struct _alpm_db_t { char *_path; alpm_pkghash_t *pkgcache; alpm_list_t *grpcache; + alpm_list_t *cache_servers; alpm_list_t *servers; const struct db_operations *ops; diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 8f25b1ef..655c0621 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -94,6 +94,7 @@ static struct server_error_count *find_server_errors(alpm_handle_t *handle, cons } } +/* skip for hard errors or too many soft errors */ static int should_skip_server(alpm_handle_t *handle, const char *server) { struct server_error_count *h; @@ -103,6 +104,17 @@ static int should_skip_server(alpm_handle_t *handle, const char *server) return 0; } +/* only skip for hard errors */ +static int should_skip_cache_server(alpm_handle_t *handle, const char *server) +{ + struct server_error_count *h; + if(server_error_limit && (h = find_server_errors(handle, server)) ) { + return h->errors < 0; + } + return 0; +} + +/* block normal servers after too many errors */ static void server_soft_error(alpm_handle_t *handle, const char *server) { struct server_error_count *h; @@ -119,6 +131,7 @@ static void server_soft_error(alpm_handle_t *handle, const char *server) } } +/* immediate block for both servers and cache servers */ static void server_hard_error(alpm_handle_t *handle, const char *server) { struct server_error_count *h; @@ -135,6 +148,31 @@ static void server_hard_error(alpm_handle_t *handle, const char *server) } } +static const char *payload_next_server(struct dload_payload *payload) +{ + while(payload->cache_servers + && should_skip_cache_server(payload->handle, payload->cache_servers->data)) { + payload->cache_servers = payload->cache_servers->next; + } + if(payload->cache_servers) { + const char *server = payload->cache_servers->data; + payload->cache_servers = payload->cache_servers->next; + payload->request_errors_ok = 1; + return server; + } + while(payload->servers + && should_skip_server(payload->handle, payload->servers->data)) { + payload->servers = payload->servers->next; + } + if(payload->servers) { + const char *server = payload->servers->data; + payload->servers = payload->servers->next; + payload->request_errors_ok = payload->errors_ok; + return server; + } + return NULL; +} + static const char *get_filename(const char *url) { char *filename = strrchr(url, '/'); @@ -414,21 +452,16 @@ static FILE *create_tempfile(struct dload_payload *payload, const char *localpat /* Return 0 if retry was successful, -1 otherwise */ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload *payload) { - const char *server; + const char *server = NULL; size_t len; struct stat st; alpm_handle_t *handle = payload->handle; - while(payload->servers && should_skip_server(handle, payload->servers->data)) { - payload->servers = payload->servers->next; - } - if(!payload->servers) { + if((server = payload_next_server(payload)) == NULL) { _alpm_log(payload->handle, ALPM_LOG_DEBUG, "%s: no more servers to retry\n", payload->remote_name); return -1; } - server = payload->servers->data; - payload->servers = payload->servers->next; /* regenerate a new fileurl */ FREE(payload->fileurl); @@ -439,7 +472,6 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload "%s: retrying from %s\n", payload->remote_name, payload->fileurl); - fflush(payload->localf); if(payload->allow_resume && stat(payload->tempfile_name, &st) == 0) { @@ -509,7 +541,7 @@ static int curl_check_finished_download(alpm_handle_t *handle, CURLM *curlm, CUR _alpm_log(handle, ALPM_LOG_DEBUG, "%s: response code %ld\n", payload->remote_name, payload->respcode); if(payload->respcode >= 400) { - if(!payload->errors_ok) { + if(!payload->request_errors_ok) { handle->pm_errno = ALPM_ERR_RETRIEVE; /* non-translated message is same as libcurl */ snprintf(payload->error_buffer, sizeof(payload->error_buffer), @@ -564,7 +596,7 @@ static int curl_check_finished_download(alpm_handle_t *handle, CURLM *curlm, CUR goto cleanup; } default: - if(!payload->errors_ok) { + if(!payload->request_errors_ok) { handle->pm_errno = ALPM_ERR_LIBCURL; _alpm_log(handle, ALPM_LOG_ERROR, _("failed retrieving file '%s' from %s : %s\n"), @@ -770,18 +802,13 @@ static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, if(payload->fileurl) { ASSERT(!payload->servers, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup)); ASSERT(!payload->filepath, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup)); + payload->request_errors_ok = payload->errors_ok; } else { - const char *server; - while(payload->servers && should_skip_server(handle, payload->servers->data)) { - payload->servers = payload->servers->next; - } + const char *server = payload_next_server(payload); - ASSERT(payload->servers, GOTO_ERR(handle, ALPM_ERR_SERVER_NONE, cleanup)); + ASSERT(server, GOTO_ERR(handle, ALPM_ERR_SERVER_NONE, cleanup)); ASSERT(payload->filepath, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup)); - server = payload->servers->data; - payload->servers = payload->servers->next; - len = strlen(server) + strlen(payload->filepath) + 2; MALLOC(payload->fileurl, len, GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); snprintf(payload->fileurl, len, "%s/%s", server, payload->filepath); @@ -946,6 +973,23 @@ static int curl_download_internal(alpm_handle_t *handle, #endif +static int payload_download_fetchcb(struct dload_payload *payload, + const char *server, const char *localpath) +{ + int ret; + char *fileurl; + alpm_handle_t *handle = payload->handle; + + size_t len = strlen(server) + strlen(payload->filepath) + 2; + MALLOC(fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); + snprintf(fileurl, len, "%s/%s", server, payload->filepath); + + ret = handle->fetchcb(handle->fetchcb_ctx, fileurl, localpath, payload->force); + free(fileurl); + + return ret; +} + /* Returns -1 if an error happened for a required file * Returns 0 if a payload was actually downloaded * Returns 1 if no files were downloaded and all errors were non-fatal @@ -971,16 +1015,11 @@ int _alpm_download(alpm_handle_t *handle, if(payload->fileurl) { ret = handle->fetchcb(handle->fetchcb_ctx, payload->fileurl, localpath, payload->force); } else { + for(s = payload->cache_servers; s && ret == -1; s = s->next) { + ret = payload_download_fetchcb(payload, s->data, localpath); + } for(s = payload->servers; s && ret == -1; s = s->next) { - const char *server = s->data; - char *fileurl; - - size_t len = strlen(server) + strlen(payload->filepath) + 2; - MALLOC(fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); - snprintf(fileurl, len, "%s/%s", server, payload->filepath); - - ret = handle->fetchcb(handle->fetchcb_ctx, fileurl, localpath, payload->force); - free(fileurl); + ret = payload_download_fetchcb(payload, s->data, localpath); } } diff --git a/lib/libalpm/dload.h b/lib/libalpm/dload.h index 9438e04a..c5210a50 100644 --- a/lib/libalpm/dload.h +++ b/lib/libalpm/dload.h @@ -37,6 +37,7 @@ struct dload_payload { */ char *fileurl; char *filepath; /* download URL path */ + alpm_list_t *cache_servers; alpm_list_t *servers; long respcode; off_t initial_size; @@ -55,6 +56,7 @@ struct dload_payload { char error_buffer[CURL_ERROR_SIZE]; FILE *localf; /* temp download file */ int signature; /* specifies if this payload is for a signature file */ + int request_errors_ok; /* per-request errors-ok */ #endif }; diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 13d2ad4f..cf436a84 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -828,6 +828,7 @@ static int download_files(alpm_handle_t *handle) FREE(payload->remote_name); FREE(payload); GOTO_ERR(handle, ALPM_ERR_MEMORY, finish)); payload->max_size = pkg->size; + payload->cache_servers = pkg->origin_data.db->cache_servers; payload->servers = pkg->origin_data.db->servers; payload->handle = handle; payload->allow_resume = 1;