add cargo config tool

Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>

12 files changed, 285 insertions, 4 deletions

diff --git a/bin/ptxdist b/bin/ptxdist
index 750048956..29c9a956c 100755
--- a/bin/ptxdist
+++ b/bin/ptxdist
@@ -787,6 +787,9 @@ Misc:
   docs-latex			generate LaTeX / PDF documentation
   export_src <target dir>	export all source archives needed for this
 				project to <target dir>
+  cargosync <pkg>		create or update the rule file for the cargo package
+				dependencies
+
 Overwrite defaults:
 
   --ptxconfig=<config>		use specified ptxconfig
@@ -1628,7 +1631,7 @@ parse_second()
 
 ######## standard target, directly into make
 
-		prepare|compile|install|targetinstall|tags)
+		prepare|compile|install|targetinstall|tags|cargosync)
 			PTXDIST_OPTIMIZE_IO=true
 			;&  # fallthrough
 		extract)
diff --git a/doc/dev_advanced_rule_files.rst b/doc/dev_advanced_rule_files.rst
index 794bd148c..9fc642090 100644
--- a/doc/dev_advanced_rule_files.rst
+++ b/doc/dev_advanced_rule_files.rst
@@ -461,3 +461,34 @@ instead.
 
 .. note:: *FOO* is still the name of our example package. It must be
   replaced by the real package name.
+
+Managing Cargo Packages
+=======================
+
+As with any other package, the correct configuration tool must be selected
+for Cargo packages:
+
+.. code-block:: make
+
+    FOO_CONF_TOOL := cargo
+
+Additional *cargo* options can be added to the configuration options like this:
+
+.. code-block:: make
+
+    FOO_CONF_OPT := \
+    	$(CROSS_CARGO_OPT) \
+    	--features ...
+
+Cargo wants to manage its own dependencies. PTXdist wants to manage all
+downloads. To make this work, PTXdist must be aware of all cargo
+dependencies. To make this possible, the package must contain a
+``Cargo.lock`` file.
+For new packages or whenever the ``Cargo.lock`` file changes, ``ptxdist
+cargosync foo`` must be called. This creates (or updates)
+``foo.cargo.make``. It is placed in the same directory as ``foo.make``.
+This introduces all dependencies from ``Cargo.lock`` as additional sources
+for the package.
+Finally, PTXdist builds all cargo packages with ``--frozen`` to ensure that
+the exact same versions are used and nothing is downloaded in the compile
+stage.
diff --git a/rules/post/ptxd_make_world_cargo.make b/rules/post/ptxd_make_world_cargo.make
new file mode 100644
index 000000000..806b5ca40
--- /dev/null
+++ b/rules/post/ptxd_make_world_cargo.make
@@ -0,0 +1,18 @@
+# -*-makefile-*-
+#
+# Copyright (C) 2021 by Michael Olbrich <m.olbrich@pengutronix.de>
+#
+# For further information about the PTXdist project and license conditions
+# see the README file.
+#
+
+world/cargo-sync = \
+	$(call world/env, $(1)) \
+	ptxd_make_world_cargo_sync
+
+$(STATEDIR)/%.cargosync:
+	@$(call targetinfo)
+	@$(call world/cargo-sync, $(PTX_MAP_TO_PACKAGE_$(*)))
+	@$(call finish)
+
+# vim: syntax=make:
diff --git a/rules/post/ptxd_make_world_common.make b/rules/post/ptxd_make_world_common.make
index 72d27f77f..43f9eede3 100644
--- a/rules/post/ptxd_make_world_common.make
+++ b/rules/post/ptxd_make_world_common.make
@@ -54,6 +54,9 @@ ptx/env = \
 	ptx_conf_opt_meson_host="$(call ptx/escape,$(HOST_MESON_OPT))"		\
 	ptx_conf_env_meson_host="$(call ptx/escape,$(HOST_ENV))"		\
 										\
+	ptx_make_opt_cargo_target="$(call ptx/escape,$(CROSS_CARGO_OPT))"	\
+	ptx_make_env_cargo_target="$(call ptx/escape,$(CROSS_CARGO_ENV))"	\
+										\
 	ptx_xpkg_extra_args=$(PTXCONF_IMAGE_XPKG_EXTRA_ARGS)
 
 world/env/impl = \
diff --git a/rules/pre/Rules.make b/rules/pre/Rules.make
index 0943a51b7..41128d3c0 100644
--- a/rules/pre/Rules.make
+++ b/rules/pre/Rules.make
@@ -261,6 +261,15 @@ CROSS_MESON_USR := \
 CROSS_MESON_ENV = \
 	$(HOST_ENV_PROGS)
 
+CROSS_CARGO_ENV := \
+	RUST_TARGET_PATH=$(PTXDIST_PLATFORMDIR)/selected_toolchain
+
+CROSS_CARGO_OPT := \
+	build \
+	--target $(PTXCONF_GNU_TARGET) \
+	--release \
+	--frozen
+
 ifdef PTXCONF_GLOBAL_IPV6
 GLOBAL_IPV6_OPTION := --enable-ipv6
 else
diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
index bcb65f83e..415372ccc 100644
--- a/scripts/lib/ptxd_lib_dgen.awk
+++ b/scripts/lib/ptxd_lib_dgen.awk
@@ -514,6 +514,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) {
 		print "$(STATEDIR)/" this_pkg ".extract: | "          "$(STATEDIR)/" this_pkg ".get"		> DGEN_DEPS_POST;
 
 	print "$(STATEDIR)/" this_pkg ".extract.post: "               "$(STATEDIR)/" this_pkg ".extract"	> DGEN_DEPS_POST;
+	print "$(STATEDIR)/" this_pkg ".cargosync: "                  "$(STATEDIR)/" this_pkg ".extract.post"	> DGEN_DEPS_POST;
 	print "$(STATEDIR)/" this_pkg ".prepare: "                    "$(STATEDIR)/" this_pkg ".extract.post"	> DGEN_DEPS_POST;
 	if (DIRTY != "true") {
 		print "$(STATEDIR)/" this_pkg ".prepare: " \
diff --git a/scripts/lib/ptxd_make_world_cargo.sh b/scripts/lib/ptxd_make_world_cargo.sh
new file mode 100644
index 000000000..24bb63091
--- /dev/null
+++ b/scripts/lib/ptxd_make_world_cargo.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+#
+# Copyright (C) 2021 by Michael Olbrich <m.olbrich@pengutronix.de>
+#
+# For further information about the PTXdist project and license conditions
+# see the README file.
+#
+
+ptxd_make_world_cargo_sync_parse() {
+    awk '
+BEGIN {
+    FS=" = "
+    name=""
+    version=""
+}
+function dump() {
+    if (name && version)
+	print name, version
+    name=""
+    version=""
+}
+/[[package]]/ {
+    dump()
+}
+$1 == "name" {
+    name=substr($2, 2, length($2)-2)
+}
+$1 == "version" {
+    version=substr($2, 2, length($2)-2)
+}
+END {
+    dump()
+}
+' Cargo.lock
+}
+export -f ptxd_make_world_cargo_sync_parse
+
+ptxd_make_world_cargo_sync_package() {
+    local url="https://crates.io/api/v1/crates/${package}/${version}/download"
+    local path="${PTXDIST_SRCDIR}/${package}-${version}.crate"
+    local PACKAGE="$(tr '[a-z]' '[A-Z]' <<< "${package}-${version}" | tr -sc '[:alnum:]' '_')"
+    PACKAGE="${PACKAGE%_}"
+
+    echo "Processing ${package} ${version} ..."
+    if [ ! -e "${path}" ]; then
+	echo "Downloading ${url} ..."
+	echo
+	ptxd_make_get "${path}" "${url}"
+    fi
+    set -- $(md5sum "${path}")
+    md5="${1}"
+    cat << EOF >&${makefilefd}
+${PKG}_${PACKAGE}_MD5		:= ${md5}
+${PKG}_${PACKAGE}_URL		:= ${url}
+${PKG}_${PACKAGE}_SOURCE	:= \$(SRCDIR)${path#${PTXDIST_SRCDIR}}
+\$(${PKG}_${PACKAGE}_SOURCE)	:= ${PKG}_${PACKAGE}
+${PKG}_SOURCES			+= \$(${PKG}_${PACKAGE}_SOURCE)
+
+EOF
+}
+export -f ptxd_make_world_cargo_sync_package
+
+ptxd_make_world_cargo_sync() {
+    local pkg_makefile_cargo package version cargofd
+    local PKG
+    local -a tmp
+
+    ptxd_make_world_init || return
+
+    if [ ! -e "${pkg_makefile}" ]; then
+	ptxd_bailout "Missing rule file for '${pkg_label}'"
+    fi
+    pkg_makefile_cargo="${pkg_makefile%.make}.cargo.make"
+    PKG="$(tr "[a-z-]" "[A-Z_]" <<< "${pkg_label}")"
+
+    cd -- "${pkg_dir}" &&
+    exec {makefilefd}> "${pkg_makefile_cargo}"
+    # copy the copyright header from the package rule file
+    sed '/^\([^#]\|$\)/Q' ${pkg_makefile} >&${makefilefd}
+
+    set -- $(md5sum "Cargo.lock")
+    cat << EOF >&${makefilefd}
+
+#
+# WARNING: This file is generated with 'ptxdist cargosync ${pkg_label}' and
+# should not be modified manually!
+#
+
+${PKG}_CONF_OPT += cargo-lock-md5 $1
+
+EOF
+
+    exec {cargofd}< <(ptxd_make_world_cargo_sync_parse) &&
+    while read package version <&${cargofd}; do
+	if [ "${package}" = "${pkg_label}" ]; then
+	    continue
+	fi
+	if [ -z "${version}" ]; then
+	    ptxd_bailout "${tmp[*]}"
+	fi
+	ptxd_make_world_cargo_sync_package
+    done
+    exec {cargofd}<&-
+
+    cat << EOF >&${makefilefd}
+
+# vim: syntax=make
+EOF
+    exec {makefilefd}<&-
+}
+export -f ptxd_make_world_cargo_sync
diff --git a/scripts/lib/ptxd_make_world_common.sh b/scripts/lib/ptxd_make_world_common.sh
index c07110d37..8b359a81b 100644
--- a/scripts/lib/ptxd_make_world_common.sh
+++ b/scripts/lib/ptxd_make_world_common.sh
@@ -317,6 +317,9 @@ ptxd_make_world_init() {
 	if [ -e "${pkg_conf_dir}/meson.build" ]; then
 	    pkg_conf_tool=${pkg_conf_tool}meson
 	fi
+	if [ -e "${pkg_conf_dir}/Cargo.toml" ]; then
+	    pkg_conf_tool=${pkg_conf_tool}cargo
+	fi
     fi
 
     case "${pkg_conf_tool}" in
@@ -355,6 +358,15 @@ ptxd_make_world_init() {
 	    pkg_conf_opt="${pkg_conf_opt:-${!conf_opt_ptr}}"
 	    pkg_conf_env="PTXDIST_ICECC= CMAKE=false CMAKE_FOR_BUILD=false ${pkg_conf_env:-${!conf_env_ptr}}"
 	    ;;
+	cargo)
+	    local make_opt_ptr="ptx_make_opt_${pkg_conf_tool}_${pkg_type}"
+	    local make_env_ptr="ptx_make_env_${pkg_conf_tool}_${pkg_type}"
+
+	    pkg_make_opt="${pkg_make_opt:-${!make_opt_ptr}}"
+	    pkg_make_env="${pkg_make_env:-${!make_env_ptr}}"
+	    pkg_cargo_home="${pkg_dir}/ptxdist-cargo-home"
+	    pkg_make_env="CARGO_HOME='${pkg_cargo_home}' ${pkg_make_env}"
+	    ;;
 	*)
 	    local conf_env_ptr="ptx_conf_env_${pkg_type}"
 	    pkg_conf_env="PTXDIST_ICECC= ${pkg_conf_env:-${!conf_env_ptr}}"
@@ -500,7 +512,7 @@ ptxd_make_world_init() {
 	    # no consistent support for parallel building
 	    pkg_make_par="${python_pkg_make_par}"
 	    ;;
-	scons)
+	scons|cargo)
 	    # only -jX is supported not other options
 	    pkg_make_par="${PTXDIST_PARALLEL_FLAGS}"
 	    ;;
diff --git a/scripts/lib/ptxd_make_world_compile.sh b/scripts/lib/ptxd_make_world_compile.sh
index 1194d24ca..d45b138e3 100644
--- a/scripts/lib/ptxd_make_world_compile.sh
+++ b/scripts/lib/ptxd_make_world_compile.sh
@@ -47,6 +47,16 @@ ptxd_make_world_compile() {
 	    "${pkg_make_opt}" \
 	    "${pkg_make_par}"
 	;;
+	cargo)
+	ptxd_eval \
+	    cd "${pkg_build_dir}" '&&' \
+	    "${pkg_path}" \
+	    "${pkg_env}" \
+	    "${pkg_make_env}" \
+	    cargo \
+	    "${pkg_make_opt}" \
+	    "${pkg_make_par}"
+	;;
 	*)
 	ptxd_eval \
 	    "${pkg_path}" \
diff --git a/scripts/lib/ptxd_make_world_extract.sh b/scripts/lib/ptxd_make_world_extract.sh
index 8ffa88bda..7e582f376 100644
--- a/scripts/lib/ptxd_make_world_extract.sh
+++ b/scripts/lib/ptxd_make_world_extract.sh
@@ -10,7 +10,7 @@
 #
 # ptxd_make_world_extract
 #
-ptxd_make_world_extract() {
+ptxd_make_world_extract_impl() {
     ptxd_make_world_init || return
 
     if [ -z "${pkg_url}" -a -z "${pkg_src}" -o -z "${pkg_dir}" ]; then
@@ -86,5 +86,51 @@ extract: pkg_extract_dir=$(ptxd_print_path ${pkg_dir})"
     ptxd_make_serialize_put
     return ${ret}
 }
+export -f ptxd_make_world_extract_impl
+
+ptxd_make_world_extract_cargo() {
+    local src
+    echo "extract: cargo dependencies:"
+    rm -rf "${pkg_cargo_home}" &&
+    mkdir -p "${pkg_cargo_home}/source" &&
+    cd "${pkg_cargo_home}/source" &&
+    for src in ${pkg_srcs}; do
+	case "${src}" in
+	*.crate)
+	    echo "extract: ${src}"
+	    tar xf "${src}" || break
+	    set -- $(sha256sum "${src}")
+	    srcdir="$(basename ${src%.crate})"
+	    if [ ! -d "${srcdir}" ]; then
+		ptxd_bailout "missing source directory '${srcdir}'"
+	    fi
+	    printf '{"files": {}, "package": "%s"}' "${1}" > "${srcdir}/.cargo-checksum.json"
+	    ;;
+	*)
+	    ;;
+	esac
+    done &&
+    cat << EOF > ${pkg_cargo_home}/config
+[source.ptxdist]
+directory = "${pkg_cargo_home}/source"
+
+[source.crates-io]
+replace-with = "ptxdist"
+local-registry = "/nonexistant"
+
+[build]
+target-dir = "${pkg_build_dir}/target"
+
+[net]
+offline = true
+EOF
+}
+export -f ptxd_make_world_extract_cargo
 
+ptxd_make_world_extract() {
+    ptxd_make_world_extract_impl &&
+    if [ "${pkg_conf_tool}" = "cargo" ]; then
+	ptxd_make_world_extract_cargo
+    fi
+}
 export -f ptxd_make_world_extract
diff --git a/scripts/lib/ptxd_make_world_install.sh b/scripts/lib/ptxd_make_world_install.sh
index 91179e3cb..89dde2227 100644
--- a/scripts/lib/ptxd_make_world_install.sh
+++ b/scripts/lib/ptxd_make_world_install.sh
@@ -113,6 +113,9 @@ ptxd_make_world_install() {
 	    "${pkg_install_opt}" \
 	)
 	;;
+	cargo)
+	ptxd_bailout "Packages that use cargo require a custom install stage"
+	;;
 	*)
 	cmd=( \
 	    "${pkg_path}" \
diff --git a/scripts/lib/ptxd_make_world_prepare.sh b/scripts/lib/ptxd_make_world_prepare.sh
index 269c05ee1..aa5f91936 100644
--- a/scripts/lib/ptxd_make_world_prepare.sh
+++ b/scripts/lib/ptxd_make_world_prepare.sh
@@ -155,6 +155,40 @@ ptxd_make_world_prepare_meson() {
 }
 export -f ptxd_make_world_prepare_meson
 
+#
+# prepare for cargo based pkgs
+#
+ptxd_make_world_prepare_cargo() {
+    local arg cargo_lock_md5
+    local -a tmp
+    local pkg_makefile_cargo="${pkg_makefile%.make}.cargo.make"
+
+    set -- ${pkg_conf_opt}
+    while [ $# -gt 0 ]; do
+	arg="${1}"
+	shift
+	case "${arg}" in
+	cargo-lock-md5)
+	    cargo_lock_md5="${1}"
+	    shift
+	    tmp=( $(md5sum Cargo.lock 2>/dev/null) )
+	    if [ "${tmp[0]}" != "${cargo_lock_md5}" ]; then
+		ptxd_bailout "Cargo.lock has changed!" \
+		    "Run 'ptxdist cargosync ${pkg_lable}' to regenerate '$(ptxd_print_path ${pkg_makefile_cargo})'."
+	    fi
+	    ;;
+	*)
+	    ptxd_bailout "unknown option '${arg}' in <PKG>_CONF_OPT!"
+	    ;;
+	esac
+    done
+    if [ -z "${cargo_lock_md5}" ]; then
+	ptxd_bailout "Cargo dependency config is missing!" \
+	    "Run 'ptxdist cargosync ${pkg_lable}' to generate '$(ptxd_print_path ${pkg_makefile_cargo})'."
+    fi
+}
+export -f ptxd_make_world_prepare_cargo
+
 ptxd_make_world_prepare_init() {
     # delete existing build_dir
     if [ -n "${pkg_build_oot}" ]; then
@@ -212,7 +246,7 @@ ptxd_make_world_prepare() {
     esac
 
     case "${pkg_conf_tool}" in
-	autoconf|cmake|qmake|kconfig|perl|meson)
+	autoconf|cmake|qmake|kconfig|perl|meson|cargo)
 	    cd -- "${pkg_build_dir}" &&
 	    ptxd_make_world_prepare_"${pkg_conf_tool}" ;;
 	python|python3|scons)