makepkg: immutable git sources by hashing the checkout content

This feature makes Git VCS build inputs immutable by adding support for
pinning a Git checkout by a hash of its content using the deterministic
export functionality `git archive`.

This feature aids packagers by allowing them to use simple and
convenient refnames (instead of full commit hashes) in the `PKGBUILD`
while still preserving security implications of immutable build inputs
using a trusted cryptographic hash function of the content.

Previously VCS source downloads have been skipped for `--geninteg` and
`--source` as both options did not need a checkout. This commit changes
this behavior by forcing the download of all sources as integrity checks
and generation requires to have an up to date state.

Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

doc/PKGBUILD.5.asciidoc

@@ -464,6 +464,9 @@ Currently makepkg supports the Bazaar, Git, Subversion, Fossil and Mercurial
 version control systems. For other version control systems, manual cloning of
 upstream repositories must be done in the `prepare()` function.
 
+Some <<VCS,VCS Sources>> like Git support pinning the checkout by a checksum of
+its content using deterministic export functionality like ``git archive''.
+
 The source URL is divided into four components:
 
 *directory*::

scripts/libmakepkg/integrity/generate_checksum.sh.in

@@ -54,7 +54,17 @@ generate_one_checksum() {
 
 		case $proto in
 			bzr|git|hg|svn)
-				sum="SKIP"
+				if declare -f "calc_checksum_$proto" > /dev/null; then
+					if ! sum=$("calc_checksum_$proto" "$netfile" "$integ"); then
+						local name
+						name=$(get_filename "$netfile")
+						error "$(gettext "Failure while calculating %s %s checksum")" "${name}" "${proto}"
+						plainerr "$(gettext "Aborting...")"
+						exit 1
+					fi
+				else
+					sum="SKIP"
+				fi
 				;;
 			*)
 				if [[ ${netfile%%::*} != *.@(sig?(n)|asc) ]]; then

scripts/libmakepkg/integrity/verify_checksum.sh.in

@@ -26,6 +26,7 @@ LIBRARY=${LIBRARY:-'@libmakepkgdir@'}
 source "$LIBRARY/util/message.sh"
 source "$LIBRARY/util/pkgbuild.sh"
 source "$LIBRARY/util/schema.sh"
+source "$LIBRARY/source.sh"
 
 check_checksums() {
 	local integ a
@@ -68,9 +69,9 @@ check_checksums() {
 }
 
 verify_integrity_one() {
-	local source_name=$1 integ=$2 expectedsum=$3
+	local source_name=$1 integ=$2 expectedsum=$3 file proto realsum
 
-	local file="$(get_filename "$source_name")"
+	file="$(get_filename "$source_name")"
 	printf '    %s ... ' "$file" >&2
 
 	if [[ $expectedsum = 'SKIP' ]]; then
@@ -78,20 +79,18 @@ verify_integrity_one() {
 		return
 	fi
 
-	if ! file="$(get_filepath "$file")"; then
+	proto="$(get_protocol "$source_name")"
-		printf '%s\n' "$(gettext "NOT FOUND")" >&2
+	if declare -f "calc_checksum_${proto}" > /dev/null; then
-		return 1
+		realsum=$("calc_checksum_${proto}" "$source_name" "$integ") || return 1
+	else
+		realsum=$(calc_checksum_file "$source_name" "$integ") || return 1
 	fi
 
-	local realsum="$("${integ}sum" "$file")"
+	if [[ ${expectedsum,,} != "$realsum" ]]; then
-	realsum="${realsum%% *}"
-	if [[ ${expectedsum,,} = "$realsum" ]]; then
-		printf '%s\n' "$(gettext "Passed")" >&2
-	else
 		printf '%s\n' "$(gettext "FAILED")" >&2
 		return 1
 	fi
-
+	printf '%s\n' "$(gettext "Passed")" >&2
 	return 0
 }
 

scripts/libmakepkg/source/file.sh.in

@@ -151,3 +151,18 @@ extract_file() {
 		chown -R 0:0 "$srcdir"
 	fi
 }
+
+calc_checksum_file() {
+	local netfile=$1 integ=$2 ret=0 file sum
+
+	if ! file="$(get_filepath "$netfile")"; then
+		printf '%s\n' "$(gettext "NOT FOUND")" >&2
+		return 1
+	fi
+
+	sum="$("${integ}sum" "$file")" || ret=1
+	sum="${sum%% *}"
+
+	printf '%s' "$sum"
+	return $ret
+}

scripts/libmakepkg/source/git.sh.in

@@ -134,3 +134,30 @@ extract_git() {
 
 	popd &>/dev/null
 }
+
+calc_checksum_git() {
+	local netfile=$1 integ=$2 ret=0 shellopts dir url fragment sum
+
+	# this function requires pipefail - save current status to restore later
+	shellopts=$(shopt -p -o pipefail)
+	shopt -s -o pipefail
+
+	dir=$(get_filepath "$netfile")
+	url=$(get_url "$netfile")
+	fragment=$(get_uri_fragment "$url")
+
+	case ${fragment%%=*} in
+		tag|commit)
+			fragval=${fragment##*=}
+			sum=$(git -c core.abbrev=no -C "$dir" archive --format tar "$fragval" | "${integ}sum" 2>&1) || ret=1
+			sum="${sum%% *}"
+			;;
+		*)
+			sum="SKIP"
+			;;
+	esac
+
+	eval "$shellopts"
+	printf '%s' "$sum"
+	return $ret
+}

scripts/makepkg.sh.in

@@ -1158,7 +1158,7 @@ if (( GENINTEG )); then
 	mkdir -p "$srcdir"
 	chmod a-s "$srcdir"
 	cd_safe "$srcdir"
-	download_sources novcs allarch >&2
+	download_sources allarch >&2
 	generate_checksums
 	exit $E_OK
 fi
@@ -1262,7 +1262,7 @@ if (( SOURCEONLY )); then
 		download_sources allarch
 	elif ( (( ! SKIPCHECKSUMS )) || \
 			( (( ! SKIPPGPCHECK )) && source_has_signatures ) ); then
-		download_sources allarch novcs
+		download_sources allarch
 	fi
 	check_source_integrity all
 	cd_safe "$startdir"