pacman/lib/libalpm/dload.c

1419 lines
41 KiB
C
Raw Normal View History

/*
* dload.c
*
* Copyright (c) 2006-2025 Pacman Development Team <pacman-dev@lists.archlinux.org>
* Copyright (c) 2002-2006 by Judd Vinet <jvinet@zeroflux.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
#include <stdbool.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <unistd.h>
#include <sys/socket.h> /* setsockopt, SO_KEEPALIVE */
download: major refactor to address lingering issues Sorry for this being such a huge patch, but I believe it is necessary for quite a few reasons which I will attempt to explain herein. I've been mulling this over for a while, but wasn't super happy with making the download interface more complex. Instead, if we carefully order things in the internal download code, we can actually make the interface simpler. 1. FS#15657 - This involves `name.db.tar.gz.part` files being left around the filesystem, and then causing all sorts of issues when someone attempts to rerun the operation they canceled. We need to ensure that if we resume a download, we are resuming it on exactly the same file; if we cannot be almost postive of that then we need to start over. 2. http://www.mail-archive.com/pacman-dev@archlinux.org/msg03536.html - Here we have a lighttpd bug to ruin the day. If we send both a Range: header and If-Modified-Since: header across the wire in a GET request, lighttpd doesn't do what we want in several cases. If the file hadn't been modified, it returns a '304 Not Modified' instead of a '206 Partial Content'. We need to do a stat (e.g. HEAD in HTTP terms) operation here, and the proceed accordingly based off the values we get back from it. 3. The mtime stuff was rather ugly, and relied on the called function to write back to a passed in reference, which isn't the greatest. Instead, use the power of the filesystem to contain this info. Every file downloaded internally is now carefully timestamped with the remote file time. This should allow the resume logic to work. In order to guarantee this, we need to implement a signal handler that catches interrupts, notifies the running code, and causes it to set the mtimes on the file. It then rethrows the signal so the pacman signal handler (or any frontend) works as expected. 4. We did a lot of funky stuff in trying to track the DB last modified time. It is a lot easier to just keep the downloaded DB file around and track the time on that rather than in a funky dot file. It also kills a lot of code. 5. For GPG verification of the databases down the road, we are going to need the DB file around for at least a short bit of time anyway, so this gets us closer to that. Signed-off-by: Dan McGee <dan@archlinux.org> [Xav: fixed printf with off_t] Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
2009-11-11 23:39:26 -06:00
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <signal.h>
#include <dirent.h>
#include <pwd.h>
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h> /* IPPROTO_TCP */
#endif
#ifdef HAVE_NETINET_TCP_H
#include <netinet/tcp.h> /* TCP_KEEPINTVL, TCP_KEEPIDLE */
#endif
#ifdef HAVE_LIBCURL
#include <curl/curl.h>
#endif
/* libalpm */
#include "dload.h"
#include "alpm_list.h"
#include "alpm.h"
#include "log.h"
#include "util.h"
#include "handle.h"
#include "sandbox.h"
static const char *get_filename(const char *url)
{
char *filename = strrchr(url, '/');
if(filename != NULL) {
return filename + 1;
}
/* no slash found, it's a filename */
return url;
}
/* prefix to avoid possible future clash with getumask(3) */
static mode_t _getumask(void)
{
mode_t mask = umask(0);
umask(mask);
return mask;
}
static int finalize_download_file(const char *filename)
{
struct stat st;
uid_t myuid = getuid();
ASSERT(filename != NULL, return -1);
ASSERT(stat(filename, &st) == 0, return -1);
if(st.st_size == 0) {
unlink(filename);
return 1;
}
if(myuid == 0) {
ASSERT(chown(filename, 0, 0) != -1, return -1);
}
ASSERT(chmod(filename, ~(_getumask()) & 0666) != -1, return -1);
return 0;
}
static FILE *create_tempfile(struct dload_payload *payload, const char *localpath)
{
int fd;
FILE *fp;
char *randpath;
size_t len;
/* create a random filename, which is opened with O_EXCL */
len = strlen(localpath) + 14 + 1;
MALLOC(randpath, len, RET_ERR(payload->handle, ALPM_ERR_MEMORY, NULL));
snprintf(randpath, len, "%salpmtmp.XXXXXX", localpath);
if((fd = mkstemp(randpath)) == -1 ||
fchmod(fd, ~(_getumask()) & 0666) ||
!(fp = fdopen(fd, payload->tempfile_openmode))) {
unlink(randpath);
close(fd);
_alpm_log(payload->handle, ALPM_LOG_ERROR,
_("failed to create temporary file for download\n"));
free(randpath);
return NULL;
}
/* fp now points to our alpmtmp.XXXXXX */
free(payload->tempfile_name);
payload->tempfile_name = randpath;
free(payload->remote_name);
STRDUP(payload->remote_name, strrchr(randpath, '/') + 1,
fclose(fp); RET_ERR(payload->handle, ALPM_ERR_MEMORY, NULL));
return fp;
}
#ifdef HAVE_LIBCURL
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
/* RFC1123 states applications should support this length */
#define HOSTNAME_SIZE 256
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm,
struct dload_payload *payload);
static int curl_gethost(const char *url, char *buffer, size_t buf_len);
/* number of "soft" errors required to blacklist a server, set to 0 to disable
* server blacklisting */
const int server_error_limit = 3;
struct server_error_count {
char server[HOSTNAME_SIZE];
int errors;
};
static struct server_error_count *find_server_errors(alpm_handle_t *handle, const char *server)
{
alpm_list_t *i;
struct server_error_count *h;
char hostname[HOSTNAME_SIZE];
/* key off the hostname because a host may serve multiple repos under
* different url's and errors are likely to be host-wide */
if(curl_gethost(server, hostname, sizeof(hostname)) != 0) {
return NULL;
}
for(i = handle->server_errors; i; i = i->next) {
h = i->data;
if(strcmp(hostname, h->server) == 0) {
return h;
}
}
if((h = calloc(sizeof(struct server_error_count), 1))
&& alpm_list_append(&handle->server_errors, h)) {
strcpy(h->server, hostname);
h->errors = 0;
return h;
} else {
free(h);
return NULL;
}
}
/* skip for hard errors or too many soft errors */
static int should_skip_server(alpm_handle_t *handle, const char *server)
{
struct server_error_count *h;
if(server_error_limit && (h = find_server_errors(handle, server)) ) {
return h->errors < 0 || h->errors >= server_error_limit;
}
return 0;
}
/* only skip for hard errors */
static int should_skip_cache_server(alpm_handle_t *handle, const char *server)
{
struct server_error_count *h;
if(server_error_limit && (h = find_server_errors(handle, server)) ) {
return h->errors < 0;
}
return 0;
}
/* block normal servers after too many errors */
static void server_soft_error(alpm_handle_t *handle, const char *server)
{
struct server_error_count *h;
if(server_error_limit
&& (h = find_server_errors(handle, server))
&& !should_skip_server(handle, server) ) {
h->errors++;
if(should_skip_server(handle, server)) {
_alpm_log(handle, ALPM_LOG_WARNING,
_("too many errors from %s, skipping for the remainder of this transaction\n"),
h->server);
}
}
}
/* immediate block for both servers and cache servers */
static void server_hard_error(alpm_handle_t *handle, const char *server)
{
struct server_error_count *h;
if(server_error_limit && (h = find_server_errors(handle, server))) {
if(h->errors != -1) {
/* always set even if already skipped for soft errors
* to disable cache servers too */
h->errors = -1;
_alpm_log(handle, ALPM_LOG_WARNING,
_("fatal error from %s, skipping for the remainder of this transaction\n"),
h->server);
}
}
}
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
static const char *payload_next_server(struct dload_payload *payload)
{
while(payload->cache_servers
&& should_skip_cache_server(payload->handle, payload->cache_servers->data)) {
payload->cache_servers = payload->cache_servers->next;
}
if(payload->cache_servers) {
const char *server = payload->cache_servers->data;
payload->cache_servers = payload->cache_servers->next;
payload->request_errors_ok = 1;
return server;
}
while(payload->servers
&& should_skip_server(payload->handle, payload->servers->data)) {
payload->servers = payload->servers->next;
}
if(payload->servers) {
const char *server = payload->servers->data;
payload->servers = payload->servers->next;
payload->request_errors_ok = payload->errors_ok;
return server;
}
return NULL;
}
enum {
ABORT_OVER_MAXFILESIZE = 1,
};
static int dload_interrupted;
download: major refactor to address lingering issues Sorry for this being such a huge patch, but I believe it is necessary for quite a few reasons which I will attempt to explain herein. I've been mulling this over for a while, but wasn't super happy with making the download interface more complex. Instead, if we carefully order things in the internal download code, we can actually make the interface simpler. 1. FS#15657 - This involves `name.db.tar.gz.part` files being left around the filesystem, and then causing all sorts of issues when someone attempts to rerun the operation they canceled. We need to ensure that if we resume a download, we are resuming it on exactly the same file; if we cannot be almost postive of that then we need to start over. 2. http://www.mail-archive.com/pacman-dev@archlinux.org/msg03536.html - Here we have a lighttpd bug to ruin the day. If we send both a Range: header and If-Modified-Since: header across the wire in a GET request, lighttpd doesn't do what we want in several cases. If the file hadn't been modified, it returns a '304 Not Modified' instead of a '206 Partial Content'. We need to do a stat (e.g. HEAD in HTTP terms) operation here, and the proceed accordingly based off the values we get back from it. 3. The mtime stuff was rather ugly, and relied on the called function to write back to a passed in reference, which isn't the greatest. Instead, use the power of the filesystem to contain this info. Every file downloaded internally is now carefully timestamped with the remote file time. This should allow the resume logic to work. In order to guarantee this, we need to implement a signal handler that catches interrupts, notifies the running code, and causes it to set the mtimes on the file. It then rethrows the signal so the pacman signal handler (or any frontend) works as expected. 4. We did a lot of funky stuff in trying to track the DB last modified time. It is a lot easier to just keep the downloaded DB file around and track the time on that rather than in a funky dot file. It also kills a lot of code. 5. For GPG verification of the databases down the road, we are going to need the DB file around for at least a short bit of time anyway, so this gets us closer to that. Signed-off-by: Dan McGee <dan@archlinux.org> [Xav: fixed printf with off_t] Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
2009-11-11 23:39:26 -06:00
static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow,
curl_off_t UNUSED ultotal, curl_off_t UNUSED ulnow)
{
struct dload_payload *payload = (struct dload_payload *)file;
off_t current_size, total_size;
alpm_download_event_progress_t cb_data = {0};
/* avoid displaying progress bar for redirects with a body */
if(payload->respcode >= 300) {
return 0;
}
/* SIGINT sent, abort by alerting curl */
if(dload_interrupted) {
return 1;
}
if(dlnow < 0 || dltotal <= 0 || dlnow > dltotal) {
/* bogus values : stop here */
return 0;
}
current_size = payload->initial_size + dlnow;
/* is our filesize still under any set limit? */
if(payload->max_size && current_size > payload->max_size) {
dload_interrupted = ABORT_OVER_MAXFILESIZE;
return 1;
}
/* none of what follows matters if the front end has no callback */
if(payload->handle->dlcb == NULL) {
return 0;
}
total_size = payload->initial_size + dltotal;
if(payload->prevprogress == total_size) {
return 0;
}
/* do NOT include initial_size since it wasn't part of the package's
* download_size (nor included in the total download size callback) */
cb_data.total = dltotal;
cb_data.downloaded = dlnow;
payload->handle->dlcb(payload->handle->dlcb_ctx,
payload->remote_name, ALPM_DOWNLOAD_PROGRESS, &cb_data);
payload->prevprogress = current_size;
return 0;
}
static int curl_gethost(const char *url, char *buffer, size_t buf_len)
{
size_t hostlen;
char *p, *q;
if(strncmp(url, "file://", 7) == 0) {
p = _("disk");
hostlen = strlen(p);
} else {
p = strstr(url, "//");
if(!p) {
return 1;
}
p += 2; /* jump over the found // */
hostlen = strcspn(p, "/");
/* there might be a user:pass@ on the URL. hide it. avoid using memrchr()
* for portability concerns. */
q = p + hostlen;
while(--q > p) {
if(*q == '@') {
break;
}
}
if(*q == '@' && p != q) {
hostlen -= q - p + 1;
p = q + 1;
}
}
if(hostlen > buf_len - 1) {
/* buffer overflow imminent */
return 1;
}
memcpy(buffer, p, hostlen);
buffer[hostlen] = '\0';
return 0;
}
static int utimes_long(const char *path, long seconds)
{
if(seconds != -1) {
struct timeval tv[2] = {
{ .tv_sec = seconds, },
{ .tv_sec = seconds, },
};
return utimes(path, tv);
}
return 0;
}
static size_t dload_parseheader_cb(void *ptr, size_t size, size_t nmemb, void *user)
{
size_t realsize = size * nmemb;
struct dload_payload *payload = (struct dload_payload *)user;
long respcode;
(void) ptr;
curl_easy_getinfo(payload->curl, CURLINFO_RESPONSE_CODE, &respcode);
if(payload->respcode != respcode) {
payload->respcode = respcode;
}
return realsize;
}
static void curl_set_handle_opts(CURL *curl, struct dload_payload *payload)
{
alpm_handle_t *handle = payload->handle;
const char *useragent = getenv("HTTP_USER_AGENT");
struct stat st;
/* the curl_easy handle is initialized with the alpm handle, so we only need
* to reset the handle's parameters for each time it's used. */
curl_easy_reset(curl);
curl_easy_setopt(curl, CURLOPT_URL, payload->fileurl);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, payload->error_buffer);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10L);
curl_easy_setopt(curl, CURLOPT_FILETIME, 1L);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, dload_progress_cb);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, (void *)payload);
if(!handle->disable_dl_timeout) {
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 10L);
}
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, dload_parseheader_cb);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)payload);
curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
curl_easy_setopt(curl, CURLOPT_TCP_KEEPIDLE, 60L);
curl_easy_setopt(curl, CURLOPT_TCP_KEEPINTVL, 60L);
curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
curl_easy_setopt(curl, CURLOPT_PRIVATE, (void *)payload);
_alpm_log(handle, ALPM_LOG_DEBUG, "%s: url is %s\n",
payload->remote_name, payload->fileurl);
if(payload->max_size) {
_alpm_log(handle, ALPM_LOG_DEBUG, "%s: maxsize %jd\n",
payload->remote_name, (intmax_t)payload->max_size);
curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE,
(curl_off_t)payload->max_size);
}
if(useragent != NULL) {
curl_easy_setopt(curl, CURLOPT_USERAGENT, useragent);
}
if(!payload->force && payload->mtime_existing_file) {
/* start from scratch, but only download if our local is out of date. */
curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
curl_easy_setopt(curl, CURLOPT_TIMEVALUE, payload->mtime_existing_file);
_alpm_log(handle, ALPM_LOG_DEBUG,
"%s: using time condition %ld\n",
payload->remote_name, (long)payload->mtime_existing_file);
} else if(stat(payload->tempfile_name, &st) == 0 && payload->allow_resume) {
/* a previous partial download exists, resume from end of file. */
payload->tempfile_openmode = "ab";
curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, (curl_off_t)st.st_size);
_alpm_log(handle, ALPM_LOG_DEBUG,
"%s: tempfile found, attempting continuation from %jd bytes\n",
payload->remote_name, (intmax_t)st.st_size);
payload->initial_size = st.st_size;
}
}
/* Return 0 if retry was successful, -1 otherwise */
static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload *payload)
{
const char *server = NULL;
size_t len;
struct stat st;
alpm_handle_t *handle = payload->handle;
if((server = payload_next_server(payload)) == NULL) {
_alpm_log(payload->handle, ALPM_LOG_DEBUG,
"%s: no more servers to retry\n", payload->remote_name);
return -1;
}
/* regenerate a new fileurl */
FREE(payload->fileurl);
len = strlen(server) + strlen(payload->filepath) + 2;
MALLOC(payload->fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1));
snprintf(payload->fileurl, len, "%s/%s", server, payload->filepath);
_alpm_log(handle, ALPM_LOG_DEBUG,
"%s: retrying from %s\n",
payload->remote_name, payload->fileurl);
fflush(payload->localf);
if(payload->allow_resume && stat(payload->tempfile_name, &st) == 0) {
/* a previous partial download exists, resume from end of file. */
payload->tempfile_openmode = "ab";
curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, (curl_off_t)st.st_size);
_alpm_log(handle, ALPM_LOG_DEBUG,
"%s: tempfile found, attempting continuation from %jd bytes\n",
payload->remote_name, (intmax_t)st.st_size);
payload->initial_size = st.st_size;
} else {
/* we keep the file for a new retry but remove its data if any */
if(ftruncate(fileno(payload->localf), 0)) {
RET_ERR(handle, ALPM_ERR_SYSTEM, -1);
}
fseek(payload->localf, 0, SEEK_SET);
}
if(handle->dlcb) {
alpm_download_event_retry_t cb_data;
cb_data.resume = payload->allow_resume;
handle->dlcb(handle->dlcb_ctx, payload->remote_name, ALPM_DOWNLOAD_RETRY, &cb_data);
}
/* Set curl with the new URL */
curl_easy_setopt(curl, CURLOPT_URL, payload->fileurl);
curl_multi_remove_handle(curlm, curl);
curl_multi_add_handle(curlm, curl);
return 0;
}
/* Returns 2 if download retry happened
* Returns 1 if the file is up-to-date
* Returns 0 if current payload is completed successfully
* Returns -1 if an error happened for a required file
* Returns -2 if an error happened for an optional file
*/
static int curl_check_finished_download(alpm_handle_t *handle, CURLM *curlm, CURLMsg *msg,
int *active_downloads_num)
{
struct dload_payload *payload = NULL;
CURL *curl = msg->easy_handle;
CURLcode curlerr;
char *effective_url;
long timecond;
curl_off_t remote_size;
curl_off_t bytes_dl = 0;
long remote_time = -1;
struct stat st;
char hostname[HOSTNAME_SIZE];
int ret = -1;
curlerr = curl_easy_getinfo(curl, CURLINFO_PRIVATE, &payload);
ASSERT(curlerr == CURLE_OK, RET_ERR(handle, ALPM_ERR_LIBCURL, -1));
curl_gethost(payload->fileurl, hostname, sizeof(hostname));
curlerr = msg->data.result;
_alpm_log(handle, ALPM_LOG_DEBUG, "%s: %s returned result %d from transfer\n",
payload->remote_name, "curl", curlerr);
/* was it a success? */
switch(curlerr) {
case CURLE_OK:
/* get http/ftp response code */
_alpm_log(handle, ALPM_LOG_DEBUG, "%s: response code %ld\n",
payload->remote_name, payload->respcode);
if(payload->respcode >= 400) {
if(!payload->request_errors_ok) {
handle->pm_errno = ALPM_ERR_RETRIEVE;
/* non-translated message is same as libcurl */
snprintf(payload->error_buffer, sizeof(payload->error_buffer),
"The requested URL returned error: %ld", payload->respcode);
_alpm_log(handle, ALPM_LOG_ERROR,
_("failed retrieving file '%s' from %s : %s\n"),
payload->remote_name, hostname, payload->error_buffer);
server_soft_error(handle, payload->fileurl);
}
fflush(payload->localf);
if(fstat(fileno(payload->localf), &st) == 0 && st.st_size != payload->initial_size) {
/* an html error page was written to the file, reset it */
if(ftruncate(fileno(payload->localf), payload->initial_size)) {
RET_ERR(handle, ALPM_ERR_SYSTEM, -1);
}
fseek(payload->localf, payload->initial_size, SEEK_SET);
}
if(curl_retry_next_server(curlm, curl, payload) == 0) {
(*active_downloads_num)++;
return 2;
} else {
payload->unlink_on_fail = 1;
goto cleanup;
}
}
break;
case CURLE_ABORTED_BY_CALLBACK:
/* handle the interrupt accordingly */
if(dload_interrupted == ABORT_OVER_MAXFILESIZE) {
curlerr = CURLE_FILESIZE_EXCEEDED;
payload->unlink_on_fail = 1;
handle->pm_errno = ALPM_ERR_LIBCURL;
_alpm_log(handle, ALPM_LOG_ERROR,
_("failed retrieving file '%s' from %s : expected download size exceeded\n"),
payload->remote_name, hostname);
server_soft_error(handle, payload->fileurl);
}
goto cleanup;
case CURLE_COULDNT_RESOLVE_HOST:
handle->pm_errno = ALPM_ERR_SERVER_BAD_URL;
_alpm_log(handle, ALPM_LOG_ERROR,
_("failed retrieving file '%s' from %s : %s\n"),
payload->remote_name, hostname, payload->error_buffer);
server_hard_error(handle, payload->fileurl);
if(curl_retry_next_server(curlm, curl, payload) == 0) {
(*active_downloads_num)++;
return 2;
} else {
goto cleanup;
}
default:
if(!payload->request_errors_ok) {
handle->pm_errno = ALPM_ERR_LIBCURL;
_alpm_log(handle, ALPM_LOG_ERROR,
_("failed retrieving file '%s' from %s : %s\n"),
payload->remote_name, hostname, payload->error_buffer);
server_soft_error(handle, payload->fileurl);
} else {
_alpm_log(handle, ALPM_LOG_DEBUG,
"failed retrieving file '%s' from %s : %s\n",
payload->remote_name, hostname, payload->error_buffer);
}
if(curl_retry_next_server(curlm, curl, payload) == 0) {
(*active_downloads_num)++;
return 2;
} else {
/* delete zero length downloads */
if(fstat(fileno(payload->localf), &st) == 0 && st.st_size == 0) {
payload->unlink_on_fail = 1;
}
goto cleanup;
}
}
/* retrieve info about the state of the transfer */
curl_easy_getinfo(curl, CURLINFO_FILETIME, &remote_time);
curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &remote_size);
curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD_T, &bytes_dl);
curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &timecond);
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url);
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
/* Let's check if client requested downloading accompanion *.sig file */
if(!payload->signature && payload->download_signature && curlerr == CURLE_OK && payload->respcode < 400) {
struct dload_payload *sig = NULL;
char *url = payload->fileurl;
char *_effective_filename;
const char *effective_filename;
char *query;
const char *dbext = alpm_option_get_dbext(handle);
const char* realname = payload->destfile_name ? payload->destfile_name : payload->tempfile_name;
int len;
STRDUP(_effective_filename, effective_url, GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup));
effective_filename = get_filename(_effective_filename);
query = strrchr(effective_filename, '?');
if(query) {
query[0] = '\0';
}
/* Only use the effective url for sig downloads if the effective_url contains .dbext or .pkg */
if(strstr(effective_filename, dbext) || strstr(effective_filename, ".pkg")) {
url = effective_url;
}
free(_effective_filename);
len = strlen(url) + 5;
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
CALLOC(sig, 1, sizeof(*sig), GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup));
MALLOC(sig->fileurl, len, FREE(sig); GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup));
snprintf(sig->fileurl, len, "%s.sig", url);
int remote_name_len = strlen(payload->remote_name) + 5;
MALLOC(sig->remote_name, remote_name_len, _alpm_dload_payload_reset(sig);
FREE(sig); GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup));
snprintf(sig->remote_name, remote_name_len, "%s.sig", payload->remote_name);
/* force the filename to be realname + ".sig" */
int destfile_name_len = strlen(realname) + 5;
MALLOC(sig->destfile_name, destfile_name_len, _alpm_dload_payload_reset(sig);
FREE(sig); GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup));
snprintf(sig->destfile_name, destfile_name_len, "%s.sig", realname);
int tempfile_name_len = strlen(realname) + 10;
MALLOC(sig->tempfile_name, tempfile_name_len, _alpm_dload_payload_reset(sig);
FREE(sig); GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup));
snprintf(sig->tempfile_name, tempfile_name_len, "%s.sig.part", realname);
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
sig->signature = 1;
sig->handle = handle;
sig->force = payload->force;
sig->unlink_on_fail = payload->unlink_on_fail;
sig->errors_ok = payload->signature_optional;
/* set hard upper limit of 16KiB */
sig->max_size = 16 * 1024;
curl_add_payload(handle, curlm, sig);
(*active_downloads_num)++;
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
}
/* time condition was met and we didn't download anything. we need to
* clean up the 0 byte .part file that's left behind. */
if(timecond == 1 && bytes_dl == 0) {
_alpm_log(handle, ALPM_LOG_DEBUG, "%s: file met time condition\n",
payload->remote_name);
ret = 1;
unlink(payload->tempfile_name);
goto cleanup;
}
/* remote_size isn't necessarily the full size of the file, just what the
* server reported as remaining to download. compare it to what curl reported
* as actually being transferred during curl_easy_perform() */
if(remote_size != -1 && bytes_dl != -1 &&
bytes_dl != remote_size) {
_alpm_log(handle, ALPM_LOG_ERROR, _("%s appears to be truncated: %jd/%jd bytes\n"),
payload->remote_name, (intmax_t)bytes_dl, (intmax_t)remote_size);
GOTO_ERR(handle, ALPM_ERR_RETRIEVE, cleanup);
}
ret = 0;
cleanup:
/* disconnect relationships from the curl handle for things that might go out
* of scope, but could still be touched on connection teardown. This really
* only applies to FTP transfers. */
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, (char *)NULL);
if(payload->localf != NULL) {
fclose(payload->localf);
utimes_long(payload->tempfile_name, remote_time);
}
if(ret == 0) {
if(payload->destfile_name) {
if(rename(payload->tempfile_name, payload->destfile_name)) {
_alpm_log(handle, ALPM_LOG_ERROR, _("could not rename %s to %s (%s)\n"),
payload->tempfile_name, payload->destfile_name, strerror(errno));
ret = -1;
}
}
}
if((ret == -1 || dload_interrupted) && payload->unlink_on_fail &&
payload->tempfile_name) {
unlink(payload->tempfile_name);
}
if(handle->dlcb) {
alpm_download_event_completed_t cb_data = {0};
cb_data.total = bytes_dl;
cb_data.result = ret;
handle->dlcb(handle->dlcb_ctx, payload->remote_name, ALPM_DOWNLOAD_COMPLETED, &cb_data);
}
curl_multi_remove_handle(curlm, curl);
curl_easy_cleanup(curl);
payload->curl = NULL;
FREE(payload->fileurl);
if(ret == -1 && payload->errors_ok) {
ret = -2;
}
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
if(payload->signature) {
/* free signature payload memory that was allocated earlier in dload.c */
_alpm_dload_payload_reset(payload);
FREE(payload);
}
return ret;
}
/* Returns 0 in case if a new download transaction has been successfully started
* Returns -1 if am error happened while starting a new download
*/
static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm,
struct dload_payload *payload)
{
size_t len;
CURL *curl = NULL;
char hostname[HOSTNAME_SIZE];
int ret = -1;
curl = curl_easy_init();
payload->curl = curl;
if(payload->fileurl) {
ASSERT(!payload->servers, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup));
ASSERT(!payload->filepath, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup));
payload->request_errors_ok = payload->errors_ok;
} else {
const char *server = payload_next_server(payload);
ASSERT(server, GOTO_ERR(handle, ALPM_ERR_SERVER_NONE, cleanup));
ASSERT(payload->filepath, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup));
len = strlen(server) + strlen(payload->filepath) + 2;
MALLOC(payload->fileurl, len, GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup));
snprintf(payload->fileurl, len, "%s/%s", server, payload->filepath);
}
payload->tempfile_openmode = "wb";
if(curl_gethost(payload->fileurl, hostname, sizeof(hostname)) != 0) {
_alpm_log(handle, ALPM_LOG_ERROR, _("url '%s' is invalid\n"), payload->fileurl);
GOTO_ERR(handle, ALPM_ERR_SERVER_BAD_URL, cleanup);
}
curl_set_handle_opts(curl, payload);
if(payload->max_size == payload->initial_size && payload->max_size != 0) {
/* .part file is complete */
ret = 0;
goto cleanup;
}
if(payload->localf == NULL) {
payload->localf = fopen(payload->tempfile_name, payload->tempfile_openmode);
if(payload->localf == NULL) {
_alpm_log(handle, ALPM_LOG_ERROR,
_("could not open file %s: %s\n"),
payload->tempfile_name, strerror(errno));
GOTO_ERR(handle, ALPM_ERR_RETRIEVE, cleanup);
}
}
_alpm_log(handle, ALPM_LOG_DEBUG,
"%s: opened tempfile for download: %s (%s)\n",
payload->remote_name,
payload->tempfile_name,
payload->tempfile_openmode);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, payload->localf);
curl_multi_add_handle(curlm, curl);
if(handle->dlcb) {
alpm_download_event_init_t cb_data = {.optional = payload->errors_ok};
handle->dlcb(handle->dlcb_ctx, payload->remote_name, ALPM_DOWNLOAD_INIT, &cb_data);
}
return 0;
cleanup:
curl_easy_cleanup(curl);
return ret;
}
/*
* Use to sort payloads by max size in descending order (largest -> smallest)
*/
static int compare_dload_payload_sizes(const void *left_ptr, const void *right_ptr)
{
struct dload_payload *left, *right;
left = (struct dload_payload *) left_ptr;
right = (struct dload_payload *) right_ptr;
return right->max_size - left->max_size;
}
/* Returns -1 if an error happened for a required file
* Returns 0 if a payload was actually downloaded
* Returns 1 if no files were downloaded and all errors were non-fatal
*/
static int curl_download_internal(alpm_handle_t *handle,
alpm_list_t *payloads /* struct dload_payload */)
{
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
int active_downloads_num = 0;
int err = 0;
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
int max_streams = handle->parallel_downloads;
int updated = 0; /* was a file actually updated */
CURLM *curlm = handle->curlm;
size_t payloads_size = alpm_list_count(payloads);
alpm_list_t *p;
/* Sort payloads by package size */
payloads = alpm_list_copy(payloads);
payloads = alpm_list_msort(payloads, payloads_size, &compare_dload_payload_sizes);
p = payloads;
while(active_downloads_num > 0 || p) {
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
CURLMcode mc;
for(; active_downloads_num < max_streams && p; active_downloads_num++) {
struct dload_payload *payload = p->data;
if(curl_add_payload(handle, curlm, payload) == 0) {
p = p->next;
} else {
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
/* The payload failed to start. Do not start any new downloads.
* Wait until all active downloads complete.
*/
_alpm_log(handle, ALPM_LOG_ERROR, _("failed to setup a download payload for %s\n"), payload->remote_name);
p = NULL;
err = -1;
}
}
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
mc = curl_multi_perform(curlm, &active_downloads_num);
if(mc == CURLM_OK) {
mc = curl_multi_wait(curlm, NULL, 0, 1000, NULL);
}
if(mc != CURLM_OK) {
_alpm_log(handle, ALPM_LOG_ERROR, _("curl returned error %d from transfer\n"), mc);
p = NULL;
err = -1;
}
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
while(true) {
int msgs_left = 0;
CURLMsg *msg = curl_multi_info_read(curlm, &msgs_left);
if(!msg) {
break;
}
if(msg->msg == CURLMSG_DONE) {
int ret = curl_check_finished_download(handle, curlm, msg,
&active_downloads_num);
if(ret == -1) {
/* if current payload failed to download then stop adding new payloads but wait for the
* current ones
*/
p = NULL;
err = -1;
} else if(ret == 0) {
updated = 1;
}
} else {
_alpm_log(handle, ALPM_LOG_ERROR, _("curl transfer error: %d\n"), msg->msg);
}
}
}
int ret = err ? -1 : updated ? 0 : 1;
_alpm_log(handle, ALPM_LOG_DEBUG, "curl_download_internal return code is %d\n", ret);
alpm_list_free(payloads);
return ret;
}
/* Download the requested files by launching a process inside a sandbox.
* Returns -1 if an error happened for a required file
* Returns 0 if a payload was actually downloaded
* Returns 1 if no files were downloaded and all errors were non-fatal
*/
static int curl_download_internal_sandboxed(alpm_handle_t *handle,
alpm_list_t *payloads /* struct dload_payload */,
const char *localpath)
{
int pid, err = 0, ret = -1, callbacks_fd[2];
sigset_t oldblock;
struct sigaction sa_ign = { .sa_handler = SIG_IGN }, oldint, oldquit;
_alpm_sandbox_callback_context callbacks_ctx;
sigemptyset(&sa_ign.sa_mask);
if(pipe(callbacks_fd) != 0) {
return -1;
}
sigaction(SIGINT, &sa_ign, &oldint);
sigaction(SIGQUIT, &sa_ign, &oldquit);
sigaddset(&sa_ign.sa_mask, SIGCHLD);
sigprocmask(SIG_BLOCK, &sa_ign.sa_mask, &oldblock);
pid = fork();
if(pid == -1) {
/* fork failed, make sure errno is preserved after cleanup */
err = errno;
}
/* child */
if(pid == 0) {
close(callbacks_fd[0]);
fcntl(callbacks_fd[1], F_SETFD, FD_CLOEXEC);
callbacks_ctx.callback_pipe = callbacks_fd[1];
alpm_option_set_logcb(handle, _alpm_sandbox_cb_log, &callbacks_ctx);
alpm_option_set_dlcb(handle, _alpm_sandbox_cb_dl, &callbacks_ctx);
alpm_option_set_fetchcb(handle, NULL, NULL);
alpm_option_set_eventcb(handle, NULL, NULL);
alpm_option_set_questioncb(handle, NULL, NULL);
alpm_option_set_progresscb(handle, NULL, NULL);
/* restore default signal handling in the child */
_alpm_reset_signals();
/* cwd to the download directory */
ret = chdir(localpath);
if(ret != 0) {
handle->pm_errno = ALPM_ERR_NOT_A_DIR;
_alpm_log(handle, ALPM_LOG_ERROR, _("could not chdir to download directory %s\n"), localpath);
ret = -1;
} else {
ret = alpm_sandbox_setup_child(handle, handle->sandboxuser, localpath, true);
if (ret != 0) {
_alpm_log(handle, ALPM_LOG_ERROR, _("switching to sandbox user '%s' failed!\n"), handle->sandboxuser);
_Exit(2);
}
ret = curl_download_internal(handle, payloads);
}
/* pass the result back to the parent */
if(ret == 0) {
/* a payload was actually downloaded */
_Exit(0);
}
else if(ret == 1) {
/* no files were downloaded and all errors were non-fatal */
_Exit(1);
}
else {
/* an error happened for a required file */
_Exit(2);
}
}
/* parent */
close(callbacks_fd[1]);
if(pid != -1) {
bool had_error = false;
while(true) {
_alpm_sandbox_callback_t callback_type;
ssize_t got = read(callbacks_fd[0], &callback_type, sizeof(callback_type));
if(got < 0 || (size_t)got != sizeof(callback_type)) {
had_error = true;
break;
}
if(callback_type == ALPM_SANDBOX_CB_DOWNLOAD) {
if(!_alpm_sandbox_process_cb_download(handle, callbacks_fd[0])) {
had_error = true;
break;
}
}
else if(callback_type == ALPM_SANDBOX_CB_LOG) {
if(!_alpm_sandbox_process_cb_log(handle, callbacks_fd[0])) {
had_error = true;
break;
}
}
}
if(had_error) {
kill(pid, SIGTERM);
}
int wret;
while((wret = waitpid(pid, &ret, 0)) == -1 && errno == EINTR);
if(wret > 0) {
if(!WIFEXITED(ret)) {
/* the child did not terminate normally */
ret = -1;
}
else {
ret = WEXITSTATUS(ret);
if(ret != 0) {
if(ret == 2) {
/* an error happened for a required file, or unexpected exit status */
handle->pm_errno = ALPM_ERR_RETRIEVE;
ret = -1;
}
else {
handle->pm_errno = ALPM_ERR_RETRIEVE;
ret = 1;
}
}
}
}
else {
/* waitpid failed */
err = errno;
}
}
close(callbacks_fd[0]);
sigaction(SIGINT, &oldint, NULL);
sigaction(SIGQUIT, &oldquit, NULL);
sigprocmask(SIG_SETMASK, &oldblock, NULL);
if(err) {
errno = err;
ret = -1;
}
return ret;
}
#endif
static int payload_download_fetchcb(struct dload_payload *payload,
const char *server, const char *localpath)
{
int ret;
char *fileurl;
alpm_handle_t *handle = payload->handle;
size_t len = strlen(server) + strlen(payload->filepath) + 2;
MALLOC(fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1));
snprintf(fileurl, len, "%s/%s", server, payload->filepath);
ret = handle->fetchcb(handle->fetchcb_ctx, fileurl, localpath, payload->force);
free(fileurl);
return ret;
}
static int move_file(const char *filepath, const char *directory)
{
ASSERT(filepath != NULL, return -1);
ASSERT(directory != NULL, return -1);
int ret = finalize_download_file(filepath);
if(ret != 0) {
return ret;
}
const char *filename = mbasename(filepath);
char *dest = _alpm_get_fullpath(directory, filename, "");
if(rename(filepath, dest)) {
FREE(dest);
return -1;
}
FREE(dest);
return 0;
}
static int finalize_download_locations(alpm_list_t *payloads, const char *localpath)
{
ASSERT(payloads != NULL, return -1);
ASSERT(localpath != NULL, return -1);
alpm_list_t *p;
int returnvalue = 0;
for(p = payloads; p; p = p->next) {
struct dload_payload *payload = p->data;
if(payload->tempfile_name) {
move_file(payload->tempfile_name, localpath);
}
if(payload->destfile_name) {
int ret = move_file(payload->destfile_name, localpath);
if(ret == -1) {
/* ignore error if the file already existed - only signature file was downloaded */
if(payload->mtime_existing_file == 0) {
returnvalue = -1;
}
}
if (payload->download_signature) {
const char sig_suffix[] = ".sig";
char *sig_filename = NULL;
size_t sig_filename_len = strlen(payload->destfile_name) + sizeof(sig_suffix);
MALLOC(sig_filename, sig_filename_len, continue);
snprintf(sig_filename, sig_filename_len, "%s%s", payload->destfile_name, sig_suffix);
move_file(sig_filename, localpath);
FREE(sig_filename);
}
}
}
return returnvalue;
}
static void prepare_resumable_downloads(alpm_list_t *payloads, const char *localpath,
const char *user)
{
struct passwd const *pw = NULL;
ASSERT(payloads != NULL, return);
ASSERT(localpath != NULL, return);
if(user != NULL) {
ASSERT((pw = getpwnam(user)) != NULL, return);
}
alpm_list_t *p;
for(p = payloads; p; p = p->next) {
struct dload_payload *payload = p->data;
if(payload->destfile_name) {
const char *destfilename = mbasename(payload->destfile_name);
char *dest = _alpm_get_fullpath(localpath, destfilename, "");
struct stat deststat;
if(stat(dest, &deststat) == 0 && deststat.st_size != 0) {
payload->mtime_existing_file = deststat.st_mtime;
}
FREE(dest);
}
if(!payload->tempfile_name) {
continue;
}
const char *filename = mbasename(payload->tempfile_name);
char *src = _alpm_get_fullpath(localpath, filename, "");
struct stat st;
if(stat(src, &st) != 0 || st.st_size == 0) {
FREE(src);
continue;
}
if(rename(src, payload->tempfile_name) != 0) {
FREE(src);
continue;
}
if(pw != NULL) {
ASSERT(chown(payload->tempfile_name, pw->pw_uid, pw->pw_gid), return);
}
FREE(src);
}
}
/* Returns -1 if an error happened for a required file
* Returns 0 if a payload was actually downloaded
* Returns 1 if no files were downloaded and all errors were non-fatal
*/
int _alpm_download(alpm_handle_t *handle,
alpm_list_t *payloads /* struct dload_payload */,
const char *localpath,
const char *temporary_localpath)
{
int ret;
prepare_resumable_downloads(payloads, localpath, handle->sandboxuser);
if(handle->fetchcb == NULL) {
#ifdef HAVE_LIBCURL
if(handle->sandboxuser) {
ret = curl_download_internal_sandboxed(handle, payloads, temporary_localpath);
} else {
ret = curl_download_internal(handle, payloads);
}
#else
RET_ERR(handle, ALPM_ERR_EXTERNAL_DOWNLOAD, -1);
#endif
} else {
alpm_list_t *p;
int updated = 0;
for(p = payloads; p; p = p->next) {
struct dload_payload *payload = p->data;
alpm_list_t *s;
ret = -1;
if(payload->fileurl) {
ret = handle->fetchcb(handle->fetchcb_ctx, payload->fileurl, temporary_localpath, payload->force);
if (ret != -1 && payload->download_signature) {
/* Download signature if requested */
char *sig_fileurl;
size_t sig_len = strlen(payload->fileurl) + 5;
int retsig = -1;
MALLOC(sig_fileurl, sig_len, RET_ERR(handle, ALPM_ERR_MEMORY, -1));
snprintf(sig_fileurl, sig_len, "%s.sig", payload->fileurl);
retsig = handle->fetchcb(handle->fetchcb_ctx, sig_fileurl, temporary_localpath, payload->force);
free(sig_fileurl);
if(!payload->signature_optional) {
ret = retsig;
}
}
} else {
for(s = payload->cache_servers; s; s = s->next) {
ret = payload_download_fetchcb(payload, s->data, temporary_localpath);
if (ret != -1) {
goto download_signature;
}
}
for(s = payload->servers; s; s = s->next) {
ret = payload_download_fetchcb(payload, s->data, temporary_localpath);
if (ret != -1) {
goto download_signature;
}
}
download_signature:
if (ret != -1 && payload->download_signature) {
/* Download signature if requested */
char *sig_fileurl;
size_t sig_len = strlen(s->data) + strlen(payload->filepath) + 6;
int retsig = -1;
MALLOC(sig_fileurl, sig_len, RET_ERR(handle, ALPM_ERR_MEMORY, -1));
snprintf(sig_fileurl, sig_len, "%s/%s.sig", (const char *)(s->data), payload->filepath);
retsig = handle->fetchcb(handle->fetchcb_ctx, sig_fileurl, temporary_localpath, payload->force);
free(sig_fileurl);
if(!payload->signature_optional) {
ret = retsig;
}
}
}
if(ret == -1 && !payload->errors_ok) {
RET_ERR(handle, ALPM_ERR_EXTERNAL_DOWNLOAD, -1);
} else if(ret == 0) {
updated = 1;
}
}
ret = updated ? 0 : 1;
}
if (finalize_download_locations(payloads, localpath) != 0 && ret == 0) {
return -1;
}
return ret;
}
static char *filecache_find_url(alpm_handle_t *handle, const char *url)
{
const char *filebase = strrchr(url, '/');
if(filebase == NULL) {
return NULL;
}
filebase++;
if(*filebase == '\0') {
return NULL;
}
return _alpm_filecache_find(handle, filebase);
}
int SYMEXPORT alpm_fetch_pkgurl(alpm_handle_t *handle, const alpm_list_t *urls,
alpm_list_t **fetched)
{
const char *cachedir;
char *temporary_cachedir = NULL;
alpm_list_t *payloads = NULL;
const alpm_list_t *i;
alpm_event_t event = {0};
CHECK_HANDLE(handle, return -1);
ASSERT(*fetched == NULL, RET_ERR(handle, ALPM_ERR_WRONG_ARGS, -1));
/* find a valid cache dir to download to */
cachedir = _alpm_filecache_setup(handle);
temporary_cachedir = _alpm_temporary_download_dir_setup(cachedir, handle->sandboxuser);
ASSERT(temporary_cachedir != NULL, return -1);
for(i = urls; i; i = i->next) {
char *url = i->data;
/* attempt to find the file in our pkgcache */
char *filepath = filecache_find_url(handle, url);
if(filepath) {
/* the file is locally cached so add it to the output right away */
alpm_list_append(fetched, filepath);
} else {
struct dload_payload *payload = NULL;
char *c;
ASSERT(url, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, err));
CALLOC(payload, 1, sizeof(*payload), GOTO_ERR(handle, ALPM_ERR_MEMORY, err));
STRDUP(payload->fileurl, url, FREE(payload); GOTO_ERR(handle, ALPM_ERR_MEMORY, err));
STRDUP(payload->remote_name, get_filename(payload->fileurl),
GOTO_ERR(handle, ALPM_ERR_MEMORY, err));
c = strrchr(url, '/');
if(c != NULL && strstr(c, ".pkg") && payload->remote_name && strlen(payload->remote_name) > 0) {
/* we probably have a usable package filename to download to */
payload->destfile_name = _alpm_get_fullpath(temporary_cachedir, payload->remote_name, "");
payload->tempfile_name = _alpm_get_fullpath(temporary_cachedir, payload->remote_name, ".part");
payload->allow_resume = 1;
if(!payload->destfile_name || !payload->tempfile_name) {
goto err;
}
} else {
/* The URL does not contain a filename, so download to a temporary location.
* We can not support resuming this kind of download; any partial transfers
* will be destroyed */
payload->unlink_on_fail = 1;
payload->tempfile_openmode = "wb";
payload->localf = create_tempfile(payload, temporary_cachedir);
if(payload->localf == NULL) {
goto err;
}
}
payload->handle = handle;
Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com> Signed-off-by: Allan McRae <allan@archlinux.org>
2020-05-12 15:26:38 -07:00
payload->download_signature = (handle->siglevel & ALPM_SIG_PACKAGE);
payload->signature_optional = (handle->siglevel & ALPM_SIG_PACKAGE_OPTIONAL);
payloads = alpm_list_add(payloads, payload);
}
}
if(payloads) {
event.type = ALPM_EVENT_PKG_RETRIEVE_START;
event.pkg_retrieve.num = alpm_list_count(payloads);
EVENT(handle, &event);
if(_alpm_download(handle, payloads, cachedir, temporary_cachedir) == -1) {
_alpm_log(handle, ALPM_LOG_WARNING, _("failed to retrieve some files\n"));
event.type = ALPM_EVENT_PKG_RETRIEVE_FAILED;
EVENT(handle, &event);
GOTO_ERR(handle, ALPM_ERR_RETRIEVE, err);
} else {
event.type = ALPM_EVENT_PKG_RETRIEVE_DONE;
EVENT(handle, &event);
}
for(i = payloads; i; i = i->next) {
struct dload_payload *payload = i->data;
char *filepath;
if(payload->destfile_name) {
const char *filename = mbasename(payload->destfile_name);
filepath = _alpm_filecache_find(handle, filename);
} else {
STRDUP(filepath, payload->tempfile_name, GOTO_ERR(handle, ALPM_ERR_MEMORY, err));
}
if(filepath) {
alpm_list_append(fetched, filepath);
} else {
_alpm_log(handle, ALPM_LOG_WARNING, _("download completed successfully but no file in the cache\n"));
GOTO_ERR(handle, ALPM_ERR_RETRIEVE, err);
}
}
alpm_list_free_inner(payloads, (alpm_list_fn_free)_alpm_dload_payload_reset);
FREELIST(payloads);
}
_alpm_remove_temporary_download_dir(temporary_cachedir);
FREE(temporary_cachedir);
return 0;
err:
alpm_list_free_inner(payloads, (alpm_list_fn_free)_alpm_dload_payload_reset);
_alpm_remove_temporary_download_dir(temporary_cachedir);
FREE(temporary_cachedir);
FREELIST(payloads);
FREELIST(*fetched);
return -1;
}
void _alpm_dload_payload_reset(struct dload_payload *payload)
{
ASSERT(payload, return);
FREE(payload->remote_name);
FREE(payload->tempfile_name);
FREE(payload->destfile_name);
FREE(payload->fileurl);
FREE(payload->filepath);
*payload = (struct dload_payload){0};
}