test: Add tests for signing and verifying with ML-DSA keys

Create mldsa-44 & mldsa-65 keys if mldsa44 can be created with the
installed version of OpenSSL. Add test cases for signing and verifying with
these types of keys.

Do not test with mldsa-87 keys since the signatures they create may be
too large for some filesystems' xattrs. On btrfs, however, it is possible
to store their large signatures.

Signed-off-by: Stefan Berger <stefanb@linux.ibm.com>

Author: stefanberger

tests/gen-keys.sh

@@ -148,6 +148,28 @@ if [ -x /opt/openssl3/bin/openssl ]; then
   done)
 fi
 
+# If creating mldsa44 key works, create all ML-DSA sizes
+if openssl genpkey -algorithm mldsa44 &>/dev/null; then
+  for mldsa in mldsa44 mldsa65; do
+    if [ "$1" = clean ] || [ "$1" = force ]; then
+      rm -f test-$mldsa.cer test-$mldsa.key test-$mldsa.pub
+    fi
+    if [ "$1" = clean ]; then
+      continue
+    fi
+    if [ ! -e test-$mldsa.key ]; then
+      log openssl req -verbose -new -nodes -utf8 -sha256 -days 10000 -batch -x509 \
+        -config test-ca.conf \
+        -newkey "$mldsa" \
+        -out test-$mldsa.cer -outform DER \
+        -keyout test-$mldsa.key
+      if [ -s test-$mldsa.key ]; then
+        log openssl pkey -in test-$mldsa.key -out test-$mldsa.pub -pubout
+      fi
+    fi
+  done
+fi
+
 # This script leaves test-ca.conf, *.cer, *.pub, *.key files for sing/verify tests.
 # They are never deleted except by `make distclean'.
 

tests/sign_verify.test

@@ -166,11 +166,13 @@ check_sign() {
     fi
 
     # Can openssl sign with this digest and key?
-    cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -sign $key -hex $FILE"
-    echo - "$cmd"
-    if ! $cmd >/dev/null; then
-      echo "${CYAN}$ALG ($key) test is skipped (openssl is unable to sign)$NORM"
-      return "$SKIP"
+    if [ "${KEY:0:10}" != "test-mldsa" ]; then
+      cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -sign $key -hex $FILE"
+      echo - "$cmd"
+      if ! $cmd >/dev/null; then
+        echo "${CYAN}$ALG ($key) test is skipped (openssl is unable to sign)$NORM"
+        return "$SKIP"
+      fi
     fi
   fi
 
@@ -213,8 +215,11 @@ check_sign() {
       verifykey=${key}
   fi
 
-  cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -verify ${verifykey} \
-	-signature $FILE.sig2 $FILE"
+  # ML-DSA uses preHashing, which does not work on command line
+  if [ "${KEY:0:10}" != "test-mldsa" ]; then
+    cmd="openssl dgst $OPENSSL_ENGINE $OPENSSL_KEYFORM -$ALG -verify ${verifykey} \
+	  -signature $FILE.sig2 $FILE"
+  fi
   echo - "$cmd"
   if ! $cmd; then
     color_red_on_failure
@@ -405,6 +410,11 @@ sign_verify  prime256v1 sha256 0x030204:K:004[345678]
 sign_verify  prime256v1 sha384 0x030205:K:004[345678]
 sign_verify  prime256v1 sha512 0x030206:K:004[345678]
 
+sign_verify  mldsa44    sha256   0x030204:K:0974
+sign_verify  mldsa44    sha512   0x030206:K:0974
+sign_verify  mldsa65    sha256   0x030204:K:0ced
+sign_verify  mldsa65    sha512   0x030206:K:0ced
+
 # If openssl 3.0 is installed, test the SM2/3 algorithm combination
 ssl_major_version=$(openssl version | sed -n 's/^OpenSSL \([^\.]\).*/\1/p')
 if [ "${ssl_major_version}" = 3 ]; then