bryson/pacman
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix several issues with xdelta

Browse source 
1) The changes to sync.c look big but there are mostly caused by
the indentation. Fix a bug where download_size == 0 because the packages and
deltas are already in the cache, but we still need to build the deltas list
and apply the deltas to create the final package.

2) Fix the gzip / md5sum issue by switching to xdelta3, disabling external
recompression and using gzip -n in pacman, and disable bsdtar compression
and using gzip -n in makepkg.

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
This commit is contained in:
Xavier Chantry 2009-02-19 19:12:34 +01:00
parent 9519d22df7
commit c8beffa790
4 changed files with 80 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/pacman.conf.5.txt
View file

@ -144,7 +144,7 @@ Options
*UseDelta*:: *UseDelta*::
Download delta files instead of complete packages if possible. Requires Download delta files instead of complete packages if possible. Requires
the xdelta program to be installed. the xdelta3 program to be installed.
*TotalDownload*:: *TotalDownload*::
When downloading, display the amount downloaded, download rate, ETA, When downloading, display the amount downloaded, download rate, ETA,

5
lib/libalpm/delta.c
View file

@ -21,6 +21,7 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <stdint.h> /* intmax_t */
#include <limits.h> #include <limits.h>
#include <sys/types.h> #include <sys/types.h>
#include <regex.h> #include <regex.h>
@ -215,13 +216,13 @@ off_t _alpm_shortest_delta_path(alpm_list_t *deltas,
return(bestsize); return(bestsize);
} }
_alpm_log(PM_LOG_DEBUG, "started delta shortest-path search\n"); _alpm_log(PM_LOG_DEBUG, "started delta shortest-path search for '%s'\n", to);
vertices = delta_graph_init(deltas); vertices = delta_graph_init(deltas);
bestsize = delta_vert(vertices, to, &bestpath); bestsize = delta_vert(vertices, to, &bestpath);
_alpm_log(PM_LOG_DEBUG, "delta shortest-path search complete\n"); _alpm_log(PM_LOG_DEBUG, "delta shortest-path search complete : '%jd'\n", (intmax_t)bestsize);
alpm_list_free_inner(vertices, _alpm_graph_free); alpm_list_free_inner(vertices, _alpm_graph_free);
alpm_list_free(vertices); alpm_list_free(vertices);

112
lib/libalpm/sync.c
View file

@ -27,6 +27,7 @@
#include <stdio.h> #include <stdio.h>
#include <fcntl.h> #include <fcntl.h>
#include <string.h> #include <string.h>
#include <stdint.h> /* intmax_t */
#include <unistd.h> #include <unistd.h>
#include <time.h> #include <time.h>
#include <dirent.h> #include <dirent.h>
@ -387,8 +388,8 @@ static int compute_download_size(pmpkg_t *newpkg)
size = alpm_pkg_get_size(newpkg); size = alpm_pkg_get_size(newpkg);
} }
_alpm_log(PM_LOG_DEBUG, "setting download size %lld for pkg %s\n", _alpm_log(PM_LOG_DEBUG, "setting download size %jd for pkg %s\n",
(long long)size, alpm_pkg_get_name(newpkg)); (intmax_t)size, alpm_pkg_get_name(newpkg));
newpkg->download_size = size; newpkg->download_size = size;
return(0); return(0);
@ -679,6 +680,12 @@ off_t SYMEXPORT alpm_pkg_download_size(pmpkg_t *newpkg)
return(newpkg->download_size); return(newpkg->download_size);
} }
static int endswith(char *filename, char *extension)
{
char *s = filename + strlen(filename) - strlen(extension);
return (strcmp(s, extension) == 0);
}
/** Applies delta files to create an upgraded package file. /** Applies delta files to create an upgraded package file.
* *
* All intermediate files are deleted, leaving only the starting and * All intermediate files are deleted, leaving only the starting and
@ -724,16 +731,18 @@ static int apply_deltas(pmtrans_t *trans)
CALLOC(to, len, sizeof(char), RET_ERR(PM_ERR_MEMORY, 1)); CALLOC(to, len, sizeof(char), RET_ERR(PM_ERR_MEMORY, 1));
snprintf(to, len, "%s/%s", cachedir, d->to); snprintf(to, len, "%s/%s", cachedir, d->to);
/* an example of the patch command: (using /cache for cachedir)
* xdelta patch /path/to/pacman_3.0.0-1_to_3.0.1-1-i686.delta \
* /path/to/pacman-3.0.0-1-i686.pkg.tar.gz \
* /cache/pacman-3.0.1-1-i686.pkg.tar.gz
*/
/* build the patch command */ /* build the patch command */
snprintf(command, PATH_MAX, "xdelta patch %s %s %s", delta, from, to); /* compression command */
char *compress = "cat";
if(endswith(to, ".gz")) {
compress = "gzip -n";
} else if(endswith(to, ".bz2")) {
compress = "bzip";
}
/* -R for disabling external recompression, -c for sending to stdout */
snprintf(command, PATH_MAX, "xdelta3 -d -q -R -c -s %s %s | %s > %s", from, delta, compress, to);
_alpm_log(PM_LOG_DEBUG, _("command: %s\n"), command); _alpm_log(PM_LOG_DEBUG, "command: %s\n", command);
EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_START, d->to, d->delta); EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_START, d->to, d->delta);
@ -847,29 +856,31 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
fname = alpm_pkg_get_filename(spkg); fname = alpm_pkg_get_filename(spkg);
ASSERT(fname != NULL, RET_ERR(PM_ERR_PKG_INVALID_NAME, -1)); ASSERT(fname != NULL, RET_ERR(PM_ERR_PKG_INVALID_NAME, -1));
if(spkg->download_size != 0) { alpm_list_t *delta_path = spkg->delta_path;
alpm_list_t *delta_path = spkg->delta_path; if(delta_path) {
if(delta_path) { /* using deltas */
alpm_list_t *dlts = NULL; alpm_list_t *dlts = NULL;
for(dlts = delta_path; dlts; dlts = dlts->next) { for(dlts = delta_path; dlts; dlts = dlts->next) {
pmdelta_t *d = dlts->data; pmdelta_t *d = dlts->data;
if(d->download_size != 0) { if(d->download_size != 0) {
/* add the delta filename to the download list if /* add the delta filename to the download list if needed */
* it's not in the cache */ files = alpm_list_add(files, strdup(d->delta));
files = alpm_list_add(files, strdup(d->delta));
}
/* keep a list of the delta files for md5sums */
deltas = alpm_list_add(deltas, d);
} }
} else { /* keep a list of all the delta files for md5sums */
/* not using deltas, so add the file to the download list */ deltas = alpm_list_add(deltas, d);
}
} else {
/* not using deltas */
if(spkg->download_size != 0) {
/* add the filename to the download list if needed */
files = alpm_list_add(files, strdup(fname)); files = alpm_list_add(files, strdup(fname));
} }
} }
} }
} }
@ -890,37 +901,34 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
handle->totaldlcb(0); handle->totaldlcb(0);
} }
if(handle->usedelta) { /* if we have deltas to work with */
if(handle->usedelta && deltas) {
int ret = 0; int ret = 0;
errors = 0;
/* Check integrity of deltas */
EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_START, NULL, NULL);
/* only output if there are deltas to work with */ for(i = deltas; i; i = i->next) {
if(deltas) { pmdelta_t *d = alpm_list_getdata(i);
errors = 0; const char *filename = alpm_delta_get_filename(d);
/* Check integrity of deltas */ const char *md5sum = alpm_delta_get_md5sum(d);
EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_START, NULL, NULL);
for(i = deltas; i; i = i->next) { if(test_md5sum(trans, filename, md5sum) != 0) {
pmdelta_t *d = alpm_list_getdata(i); errors++;
const char *filename = alpm_delta_get_filename(d); *data = alpm_list_add(*data, strdup(filename));
const char *md5sum = alpm_delta_get_md5sum(d);
if(test_md5sum(trans, filename, md5sum) != 0) {
errors++;
*data = alpm_list_add(*data, strdup(filename));
}
} }
if(errors) {
pm_errno = PM_ERR_DLT_INVALID;
goto error;
}
EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_DONE, NULL, NULL);
/* Use the deltas to generate the packages */
EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_START, NULL, NULL);
ret = apply_deltas(trans);
EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_DONE, NULL, NULL);
} }
if(errors) {
pm_errno = PM_ERR_DLT_INVALID;
goto error;
}
EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_DONE, NULL, NULL);
/* Use the deltas to generate the packages */
EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_START, NULL, NULL);
ret = apply_deltas(trans);
EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_DONE, NULL, NULL);
if(ret) { if(ret) {
pm_errno = PM_ERR_DLT_PATCHFAILED; pm_errno = PM_ERR_DLT_PATCHFAILED;
goto error; goto error;

23
scripts/makepkg.sh.in
View file

@ -963,25 +963,34 @@ create_package() {
# tar it up # tar it up
msg2 "$(gettext "Compressing package...")" msg2 "$(gettext "Compressing package...")"
local TAR_OPT
case "$PKGEXT" in case "$PKGEXT" in
*tar.gz) TAR_OPT="z" ;; *tar.gz) EXT=${PKGEXT%.gz} ;;
*tar.bz2) TAR_OPT="j" ;; *tar.bz2) EXT=${PKGEXT%.bz2} ;;
*) warning "$(gettext "'%s' is not a valid archive extension.")" \ *) warning "$(gettext "'%s' is not a valid archive extension.")" \
"$PKGEXT" ;; "$PKGEXT" ; EXT=$PKGEXT ;;
esac esac
local pkg_file="$PKGDEST/${nameofpkg}-${pkgver}-${pkgrel}-${CARCH}${EXT}"
local pkg_file="$PKGDEST/${nameofpkg}-${pkgver}-${pkgrel}-${CARCH}${PKGEXT}" local ret=0
# when fileglobbing, we want * in an empty directory to expand to # when fileglobbing, we want * in an empty directory to expand to
# the null string rather than itself # the null string rather than itself
shopt -s nullglob shopt -s nullglob
bsdtar -cf - $comp_files * > "$pkg_file" || ret=$?
shopt -u nullglob
if ! bsdtar -c${TAR_OPT}f "$pkg_file" $comp_files *; then if [ $ret -eq 0 ]; then
case "$PKGEXT" in
*tar.gz) gzip -f -n "$pkg_file" ;;
*tar.bz2) bzip2 -f "$pkg_file" ;;
esac
ret=$?
fi
if [ $ret -ne 0 ]; then
error "$(gettext "Failed to create package file.")" error "$(gettext "Failed to create package file.")"
exit 1 # TODO: error code exit 1 # TODO: error code
fi fi
shopt -u nullglob
} }
create_srcpackage() { create_srcpackage() {