ptxdist: add urlcheck command

Checking if an archive is available without downloading it is difficult and
not 100 percent reliable. For best result curl needs to fail for error http
status codes (--fail) and follow redirects (--location). A HEAD request
(--head) is used to avoid downloading the whole file, but we send a GET
anyways (--request GET) because some servers don't allow HEAD requests.

Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>

2 files changed, 63 insertions, 18 deletions

diff --git a/scripts/lib/ptxd_make_get.sh b/scripts/lib/ptxd_make_get.sh
index fd8ad333a..2bac97817 100644
--- a/scripts/lib/ptxd_make_get.sh
+++ b/scripts/lib/ptxd_make_get.sh
@@ -16,7 +16,8 @@
 # ${opts[]}	: an array of options
 #
 ptxd_make_get_http() {
-	local temp_file
+	local -a curl_opts
+	local temp_file temp_header
 	set -- "${opts[@]}"
 	unset opts
 
@@ -28,13 +29,21 @@ ptxd_make_get_http() {
 		shift
 
 		case "${opt}" in
-			no-check-certificate|no-proxy)
+			no-check-certificate)
 				opts[${#opts[@]}]="--${opt}"
+				curl_opts[${#curl_opts[@]}]="--insecure"
+				;;
+			no-proxy)
+				opts[${#opts[@]}]="--${opt}"
+				curl_opts[${#curl_opts[@]}]="--noproxy"
+				curl_opts[${#curl_opts[@]}]="*"
 				;;
 			cookie:*)
 				opts[${#opts[@]}]="--no-cookies"
 				opts[${#opts[@]}]="--header"
 				opts[${#opts[@]}]="Cookie: ${opt#cookie:}"
+				curl_opts[${#curl_opts[@]}]="--cookie"
+				curl_opts[${#curl_opts[@]}]="${opt#cookie:}"
 				;;
 			*)
 				ptxd_bailout "invalid option '${opt}' to ${FUNCNAME}"
@@ -54,23 +63,47 @@ ptxd_make_get_http() {
 
 	temp_file="$(mktemp "${path}.XXXXXXXXXX")" || ptxd_bailout "failed to create tempfile"
 	ptxd_make_serialize_take
-	wget \
-	    --passive-ftp \
-	    --progress=bar:force \
-	    --timeout=30 \
-	    --tries=5 \
-	    --user-agent="PTXdist ${PTXDIST_VERSION_FULL}" \
-	    ${PTXDIST_QUIET:+--quiet} \
-	    "${opts[@]}" \
-	    -O "${temp_file}" \
-	    "${url}" && {
-		chmod 644 -- "${temp_file}" &&
-		file "${temp_file}" | grep -vq " HTML " &&
-		touch -- "${temp_file}" &&
-		mv -- "${temp_file}" "${path}"
+	if [ "${ptxd_make_get_dryrun}" != "y" ]; then
+	    wget \
+		--passive-ftp \
+		--progress=bar:force \
+		--timeout=30 \
+		--tries=5 \
+		--user-agent="PTXdist ${PTXDIST_VERSION_FULL}" \
+		${PTXDIST_QUIET:+--quiet} \
+		"${opts[@]}" \
+		-O "${temp_file}" \
+		"${url}" && {
+		    chmod 644 -- "${temp_file}" &&
+		    file "${temp_file}" | grep -vq " HTML " &&
+		    touch -- "${temp_file}" &&
+		    mv -- "${temp_file}" "${path}"
+		    ptxd_make_serialize_put
+		    return
+	    }
+	else
+	    echo "Checking URL '${url}'..."
+	    temp_header="$(mktemp "${PTXDIST_TEMPDIR}/urlcheck.XXXXXX")" || ptxd_bailout "failed to create tempfile"
+	    curl \
+		--ftp-pasv \
+		--connect-timeout 30 \
+		--retry 5 \
+		--user-agent "PTXdist ${PTXDIST_VERSION_FULL}" \
+		${PTXDIST_QUIET:+--silent} \
+		"${curl_opts[@]}" \
+		-o /dev/null \
+		--dump-header "${temp_header}" \
+		--fail \
+		--location \
+		--head \
+		--request GET \
+		"${url}" &&
+		if grep -q "content-type:text/html" "${temp_header}"; then
+		    ptxd_bailout "Got HTML file"
+		fi
 		ptxd_make_serialize_put
 		return
-	}
+	fi &&
 	ptxd_make_serialize_put
 
 	rm -f -- "${temp_file}"
@@ -128,6 +161,12 @@ ptxd_make_get_git() {
 	fi
 
 	ptxd_make_serialize_take
+	if [ "${ptxd_make_get_dryrun}" = "y" ]; then
+	    echo "Checking URL '${url}'..."
+	    git ls-remote --quiet "${url}" HEAD > /dev/null
+	    ptxd_make_serialize_put
+	    return
+	fi
 	echo "${PROMPT}git: fetching '${url} into '${mirror}'..."
 	if [ ! -d "${mirror}" ]; then
 		git init --bare --shared "${mirror}"
@@ -211,6 +250,12 @@ ptxd_make_get_svn() {
 	fi
 
 	ptxd_make_serialize_take
+	if [ "${ptxd_make_get_dryrun}" = "y" ]; then
+	    echo "Checking URL '${url}'..."
+	    svn ls "${url}" > /dev/null
+	    ptxd_make_serialize_put
+	    return
+	fi
 	echo "${PROMPT}svn: fetching '${url} into '${mirror}'..."
 	if [ ! -d "${mirror}" ]; then
 		svn checkout -r ${rev} "${url}" "${mirror}"
diff --git a/scripts/lib/ptxd_make_world_get.sh b/scripts/lib/ptxd_make_world_get.sh
index 973afa65b..a2a365585 100644
--- a/scripts/lib/ptxd_make_world_get.sh
+++ b/scripts/lib/ptxd_make_world_get.sh
@@ -21,7 +21,7 @@ ptxd_make_world_get() {
 	    ;;
     esac
 
-    if [ -n "${pkg_src}" -a \! -e "${pkg_src}" ]; then
+    if [ -n "${pkg_src}" -a \( \! -e "${pkg_src}" -o "${ptxd_make_get_dryrun}" = "y" \) ]; then
 	ptxd_make_get "${pkg_src}" "${pkg_url}"
     fi
 }