alpm: add cache server support

Cache servers differ from regular servers in that they do not produce
warnings and are not removed from the server pool for "soft errors"
(i.e. the server was reachable, but the download failed) and they are
not used for databases.  If a host is used for both a cache server and a
regular server, it may still be removed from the server pool for soft
errors that occur when used as cache server and removal from the server
pool for soft errors will not affect future attempted use as a cache
server.

Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com>

lib/libalpm/alpm.h

@@ -1343,6 +1343,34 @@ int alpm_db_add_server(alpm_db_t *db, const char *url);
  */
 int alpm_db_remove_server(alpm_db_t *db, const char *url);
 
+/** Get the list of cache servers assigned to this db.
+ * @param db pointer to the database to get the servers from
+ * @return a char* list of servers
+ */
+alpm_list_t *alpm_db_get_cache_servers(const alpm_db_t *db);
+
+/** Sets the list of cache servers for the database to use.
+ * @param db the database to set the servers. The list will be duped and
+ * the original will still need to be freed by the caller.
+ * @param servers a char* list of servers.
+ */
+int alpm_db_set_cache_servers(alpm_db_t *db, alpm_list_t *servers);
+
+/** Add a download cache server to a database.
+ * @param db database pointer
+ * @param url url of the server
+ * @return 0 on success, -1 on error (pm_errno is set accordingly)
+ */
+int alpm_db_add_cache_server(alpm_db_t *db, const char *url);
+
+/** Remove a download cache server from a database.
+ * @param db database pointer
+ * @param url url of the server
+ * @return 0 on success, 1 on server not present,
+ * -1 on error (pm_errno is set accordingly)
+ */
+int alpm_db_remove_cache_server(alpm_db_t *db, const char *url);
+
 /* End of server accessors */
 /** @} */
 

lib/libalpm/db.c

@@ -131,6 +131,26 @@ int SYMEXPORT alpm_db_unregister(alpm_db_t *db)
 	return 0;
 }
 
+alpm_list_t SYMEXPORT *alpm_db_get_cache_servers(const alpm_db_t *db)
+{
+	ASSERT(db != NULL, return NULL);
+	return db->cache_servers;
+}
+
+int SYMEXPORT alpm_db_set_cache_servers(alpm_db_t *db, alpm_list_t *cache_servers)
+{
+	alpm_list_t *i;
+	ASSERT(db != NULL, return -1);
+	FREELIST(db->cache_servers);
+	for(i = cache_servers; i; i = i->next) {
+		char *url = i->data;
+		if(alpm_db_add_cache_server(db, url) != 0) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
 alpm_list_t SYMEXPORT *alpm_db_get_servers(const alpm_db_t *db)
 {
 	ASSERT(db != NULL, return NULL);
@@ -164,6 +184,26 @@ static char *sanitize_url(const char *url)
 	return newurl;
 }
 
+int SYMEXPORT alpm_db_add_cache_server(alpm_db_t *db, const char *url)
+{
+	char *newurl;
+
+	/* Sanity checks */
+	ASSERT(db != NULL, return -1);
+	db->handle->pm_errno = ALPM_ERR_OK;
+	ASSERT(url != NULL && strlen(url) != 0, RET_ERR(db->handle, ALPM_ERR_WRONG_ARGS, -1));
+
+	newurl = sanitize_url(url);
+	if(!newurl) {
+		return -1;
+	}
+	db->cache_servers = alpm_list_add(db->cache_servers, newurl);
+	_alpm_log(db->handle, ALPM_LOG_DEBUG, "adding new cache server URL to database '%s': %s\n",
+			db->treename, newurl);
+
+	return 0;
+}
+
 int SYMEXPORT alpm_db_add_server(alpm_db_t *db, const char *url)
 {
 	char *newurl;
@@ -184,6 +224,34 @@ int SYMEXPORT alpm_db_add_server(alpm_db_t *db, const char *url)
 	return 0;
 }
 
+int SYMEXPORT alpm_db_remove_cache_server(alpm_db_t *db, const char *url)
+{
+	char *newurl, *vdata = NULL;
+	int ret = 1;
+
+	/* Sanity checks */
+	ASSERT(db != NULL, return -1);
+	db->handle->pm_errno = ALPM_ERR_OK;
+	ASSERT(url != NULL && strlen(url) != 0, RET_ERR(db->handle, ALPM_ERR_WRONG_ARGS, -1));
+
+	newurl = sanitize_url(url);
+	if(!newurl) {
+		return -1;
+	}
+
+	db->cache_servers = alpm_list_remove_str(db->cache_servers, newurl, &vdata);
+
+	if(vdata) {
+		_alpm_log(db->handle, ALPM_LOG_DEBUG, "removed cache server URL from database '%s': %s\n",
+				db->treename, newurl);
+		free(vdata);
+		ret = 0;
+	}
+
+	free(newurl);
+	return ret;
+}
+
 int SYMEXPORT alpm_db_remove_server(alpm_db_t *db, const char *url)
 {
 	char *newurl, *vdata = NULL;
@@ -328,6 +396,7 @@ void _alpm_db_free(alpm_db_t *db)
 	/* cleanup pkgcache */
 	_alpm_db_free_pkgcache(db);
 	/* cleanup server list */
+	FREELIST(db->cache_servers);
 	FREELIST(db->servers);
 	FREE(db->_path);
 	FREE(db->treename);

lib/libalpm/db.h

@@ -69,6 +69,7 @@ struct _alpm_db_t {
 	char *_path;
 	alpm_pkghash_t *pkgcache;
 	alpm_list_t *grpcache;
+	alpm_list_t *cache_servers;
 	alpm_list_t *servers;
 	const struct db_operations *ops;
 

lib/libalpm/dload.c

@@ -94,6 +94,7 @@ static struct server_error_count *find_server_errors(alpm_handle_t *handle, cons
 	}
 }
 
+/* skip for hard errors or too many soft errors */
 static int should_skip_server(alpm_handle_t *handle, const char *server)
 {
 	struct server_error_count *h;
@@ -103,6 +104,17 @@ static int should_skip_server(alpm_handle_t *handle, const char *server)
 	return 0;
 }
 
+/* only skip for hard errors */
+static int should_skip_cache_server(alpm_handle_t *handle, const char *server)
+{
+	struct server_error_count *h;
+	if(server_error_limit && (h = find_server_errors(handle, server)) ) {
+		return h->errors < 0;
+	}
+	return 0;
+}
+
+/* block normal servers after too many errors */
 static void server_soft_error(alpm_handle_t *handle, const char *server)
 {
 	struct server_error_count *h;
@@ -119,6 +131,7 @@ static void server_soft_error(alpm_handle_t *handle, const char *server)
 	}
 }
 
+/* immediate block for both servers and cache servers */
 static void server_hard_error(alpm_handle_t *handle, const char *server)
 {
 	struct server_error_count *h;
@@ -135,6 +148,31 @@ static void server_hard_error(alpm_handle_t *handle, const char *server)
 	}
 }
 
+static const char *payload_next_server(struct dload_payload *payload)
+{
+	while(payload->cache_servers
+			&& should_skip_cache_server(payload->handle, payload->cache_servers->data)) {
+		payload->cache_servers = payload->cache_servers->next;
+	}
+	if(payload->cache_servers) {
+		const char *server = payload->cache_servers->data;
+		payload->cache_servers = payload->cache_servers->next;
+		payload->request_errors_ok = 1;
+		return server;
+	}
+	while(payload->servers
+			&& should_skip_server(payload->handle, payload->servers->data)) {
+		payload->servers = payload->servers->next;
+	}
+	if(payload->servers) {
+		const char *server = payload->servers->data;
+		payload->servers = payload->servers->next;
+		payload->request_errors_ok = payload->errors_ok;
+		return server;
+	}
+	return NULL;
+}
+
 static const char *get_filename(const char *url)
 {
 	char *filename = strrchr(url, '/');
@@ -414,21 +452,16 @@ static FILE *create_tempfile(struct dload_payload *payload, const char *localpat
 /* Return 0 if retry was successful, -1 otherwise */
 static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload *payload)
 {
-	const char *server;
+	const char *server = NULL;
 	size_t len;
 	struct stat st;
 	alpm_handle_t *handle = payload->handle;
 
-	while(payload->servers && should_skip_server(handle, payload->servers->data)) {
+	if((server = payload_next_server(payload)) == NULL) {
-		payload->servers = payload->servers->next;
-	}
-	if(!payload->servers) {
 		_alpm_log(payload->handle, ALPM_LOG_DEBUG,
 				"%s: no more servers to retry\n", payload->remote_name);
 		return -1;
 	}
-	server = payload->servers->data;
-	payload->servers = payload->servers->next;
 
 	/* regenerate a new fileurl */
 	FREE(payload->fileurl);
@@ -439,7 +472,6 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload
 			"%s: retrying from %s\n",
 			payload->remote_name, payload->fileurl);
 
-
 	fflush(payload->localf);
 
 	if(payload->allow_resume && stat(payload->tempfile_name, &st) == 0) {
@@ -509,7 +541,7 @@ static int curl_check_finished_download(alpm_handle_t *handle, CURLM *curlm, CUR
 			_alpm_log(handle, ALPM_LOG_DEBUG, "%s: response code %ld\n",
 					payload->remote_name, payload->respcode);
 			if(payload->respcode >= 400) {
-				if(!payload->errors_ok) {
+				if(!payload->request_errors_ok) {
 					handle->pm_errno = ALPM_ERR_RETRIEVE;
 					/* non-translated message is same as libcurl */
 					snprintf(payload->error_buffer, sizeof(payload->error_buffer),
@@ -564,7 +596,7 @@ static int curl_check_finished_download(alpm_handle_t *handle, CURLM *curlm, CUR
 				goto cleanup;
 			}
 		default:
-			if(!payload->errors_ok) {
+			if(!payload->request_errors_ok) {
 				handle->pm_errno = ALPM_ERR_LIBCURL;
 				_alpm_log(handle, ALPM_LOG_ERROR,
 						_("failed retrieving file '%s' from %s : %s\n"),
@@ -770,18 +802,13 @@ static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm,
 	if(payload->fileurl) {
 		ASSERT(!payload->servers, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup));
 		ASSERT(!payload->filepath, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup));
+		payload->request_errors_ok = payload->errors_ok;
 	} else {
-		const char *server;
+		const char *server = payload_next_server(payload);
-		while(payload->servers && should_skip_server(handle, payload->servers->data)) {
-			payload->servers = payload->servers->next;
-		}
 
-		ASSERT(payload->servers, GOTO_ERR(handle, ALPM_ERR_SERVER_NONE, cleanup));
+		ASSERT(server, GOTO_ERR(handle, ALPM_ERR_SERVER_NONE, cleanup));
 		ASSERT(payload->filepath, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup));
 
-		server = payload->servers->data;
-		payload->servers = payload->servers->next;
-
 		len = strlen(server) + strlen(payload->filepath) + 2;
 		MALLOC(payload->fileurl, len, GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup));
 		snprintf(payload->fileurl, len, "%s/%s", server, payload->filepath);
@@ -946,6 +973,23 @@ static int curl_download_internal(alpm_handle_t *handle,
 
 #endif
 
+static int payload_download_fetchcb(struct dload_payload *payload,
+		const char *server, const char *localpath)
+{
+	int ret;
+	char *fileurl;
+	alpm_handle_t *handle = payload->handle;
+
+	size_t len = strlen(server) + strlen(payload->filepath) + 2;
+	MALLOC(fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1));
+	snprintf(fileurl, len, "%s/%s", server, payload->filepath);
+
+	ret = handle->fetchcb(handle->fetchcb_ctx, fileurl, localpath, payload->force);
+	free(fileurl);
+
+	return ret;
+}
+
 /* Returns -1 if an error happened for a required file
  * Returns 0 if a payload was actually downloaded
  * Returns 1 if no files were downloaded and all errors were non-fatal
@@ -971,16 +1015,11 @@ int _alpm_download(alpm_handle_t *handle,
 			if(payload->fileurl) {
 				ret = handle->fetchcb(handle->fetchcb_ctx, payload->fileurl, localpath, payload->force);
 			} else {
+				for(s = payload->cache_servers; s && ret == -1; s = s->next) {
+					ret = payload_download_fetchcb(payload, s->data, localpath);
+				}
 				for(s = payload->servers; s && ret == -1; s = s->next) {
-					const char *server = s->data;
+					ret = payload_download_fetchcb(payload, s->data, localpath);
-					char *fileurl;
-
-					size_t len = strlen(server) + strlen(payload->filepath) + 2;
-					MALLOC(fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1));
-					snprintf(fileurl, len, "%s/%s", server, payload->filepath);
-
-					ret = handle->fetchcb(handle->fetchcb_ctx, fileurl, localpath, payload->force);
-					free(fileurl);
 				}
 			}
 

lib/libalpm/dload.h

@@ -37,6 +37,7 @@ struct dload_payload {
 	 */
 	char *fileurl;
 	char *filepath; /* download URL path */
+	alpm_list_t *cache_servers;
 	alpm_list_t *servers;
 	long respcode;
 	off_t initial_size;
@@ -55,6 +56,7 @@ struct dload_payload {
 	char error_buffer[CURL_ERROR_SIZE];
 	FILE *localf; /* temp download file */
 	int signature; /* specifies if this payload is for a signature file */
+	int request_errors_ok; /* per-request errors-ok */
 #endif
 };
 

lib/libalpm/sync.c

@@ -828,6 +828,7 @@ static int download_files(alpm_handle_t *handle)
 				FREE(payload->remote_name); FREE(payload);
 				GOTO_ERR(handle, ALPM_ERR_MEMORY, finish));
 			payload->max_size = pkg->size;
+			payload->cache_servers = pkg->origin_data.db->cache_servers;
 			payload->servers = pkg->origin_data.db->servers;
 			payload->handle = handle;
 			payload->allow_resume = 1;