test: Add tests for signing and verifying with ML-DSA keys

Create ML-DSA-44 & ML-DSA-65 keys if ML-DSA-44 can be created with the
installed version of OpenSSL. Add test cases for signing and verifying with
these types of keys.

Do not test with ML-DSA-87 keys since the signatures they create may be
too large for some filesystems' xattrs. On Btrfs, however, it would be
possible to store the large signatures.

Signed-off-by: Stefan Berger <stefanb@linux.ibm.com>

Author: stefanberger

tests/gen-keys.sh

@@ -148,6 +148,28 @@ if [ -x /opt/openssl3/bin/openssl ]; then
   done)
 fi
 
+# If creating mldsa44 key works, create all ML-DSA sizes
+if openssl genpkey -algorithm mldsa44 &>/dev/null; then
+  for mldsa in mldsa44 mldsa65; do
+    if [ "$1" = clean ] || [ "$1" = force ]; then
+      rm -f test-$mldsa.cer test-$mldsa.key test-$mldsa.pub
+    fi
+    if [ "$1" = clean ]; then
+      continue
+    fi
+    if [ ! -e test-$mldsa.key ]; then
+      log openssl req -verbose -new -nodes -utf8 -sha256 -days 10000 -batch -x509 \
+        -config test-ca.conf \
+        -newkey "$mldsa" \
+        -out test-$mldsa.cer -outform DER \
+        -keyout test-$mldsa.key
+      if [ -s test-$mldsa.key ]; then
+        log openssl pkey -in test-$mldsa.key -out test-$mldsa.pub -pubout
+      fi
+    fi
+  done
+fi
+
 # This script leaves test-ca.conf, *.cer, *.pub, *.key files for sing/verify tests.
 # They are never deleted except by `make distclean'.
 

tests/sign_verify.test

@@ -166,8 +166,14 @@ check_sign() {
     fi
 
     # Can openssl sign with this digest and key?
-    cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -sign $key -hex $FILE"
-    echo - "$cmd"
+    case "${KEY:0:10}" in
+    "test-mldsa")
+      cmd="openssl pkeyutl -sign -inkey $key -in $FILE"
+      echo >> "$FILE" # need at least 1 byte in the file for signing to work
+      ;;
+    *)
+      cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -sign $key -hex $FILE"
+    esac
     if ! $cmd >/dev/null; then
       echo "${CYAN}$ALG ($key) test is skipped (openssl is unable to sign)$NORM"
       return "$SKIP"
@@ -219,8 +225,18 @@ check_sign() {
     echo -en "\x${PREFIX:2:2}\x${PREFIX:6:2}" > "$FILE.tmp"
     # shellcheck disable=SC2086
     openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -"$ALG" -binary "$FILE" >> "$FILE.tmp"
-    cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -verify ${verifykey} \
-	  -signature $FILE.sig2 $FILE.tmp"
+
+    case "${KEY:0:10}" in
+    "test-mldsa")
+      # ML-DSA needs the ima_file_id as input
+      openssl dgst -$ALG -binary $FILE > $FILE.hash
+      cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM       -verify ${verifykey} \
+	    -signature $FILE.sig2 $FILE.tmp"
+      ;;
+    *)
+      cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -verify ${verifykey} \
+	    -signature $FILE.sig2 $FILE.tmp"
+    esac
     sigver=3
   else
     cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -verify ${verifykey} \
@@ -424,6 +440,11 @@ sign_verify_ima  prime256v1 sha256   0x030304:K:004[345678] --v3
 sign_verify_ima  prime256v1 sha384   0x030305:K:004[345678] --v3
 sign_verify_ima  prime256v1 sha512   0x030306:K:004[345678] --v3
 
+sign_verify  mldsa44    sha256   0x030304:K:0974 --v3
+sign_verify  mldsa44    sha512   0x030306:K:0974 --v3
+sign_verify  mldsa65    sha256   0x030304:K:0ced --v3
+sign_verify  mldsa65    sha512   0x030306:K:0ced --v3
+
 # If openssl 3.0 is installed, test the SM2/3 algorithm combination
 ssl_major_version=$(openssl version | sed -n 's/^OpenSSL \([^\.]\).*/\1/p')
 if [ "${ssl_major_version}" = 3 ]; then