Merge pull request #9784 from dgarske/async_cryptocb

Fixes and tests for async and crypto callbacks

Author: douzzer

.github/workflows/async-examples.yml

@@ -18,24 +18,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        async_mode: ['sw', 'cryptocb']
         extra_cflags:
           - ''
           - '-DWOLFSSL_SMALL_CERT_VERIFY'
           - '-DWOLFSSL_STATIC_MEMORY'
-    name: Async Examples (${{ matrix.extra_cflags || 'default' }})
+    name: Async Examples (${{ matrix.async_mode }}, ${{ matrix.extra_cflags || 'default' }})
     steps:
       - uses: actions/checkout@v4
         name: Checkout wolfSSL
 
       - name: Build async examples (no configure)
         run: |
           make -C examples/async clean
-          make -C examples/async EXTRA_CFLAGS="${{ matrix.extra_cflags }}"
+          make -C examples/async ASYNC_MODE=${{ matrix.async_mode }} EXTRA_CFLAGS="${{ matrix.extra_cflags }}"
 
       - name: Run async examples
         run: |
           set -euo pipefail
 
+          ASYNC_MODE="${{ matrix.async_mode }}"
           MIN_PENDING=100
 
           run_pair() {
@@ -63,16 +65,21 @@ jobs:
               return 1
             fi
 
-            # Validate WC_PENDING_E count is a proper value
-            local count
-            count=$(awk '/WC_PENDING_E count:/ {print $NF}' \
-              "/tmp/async_client_${label}.log")
-            if [ -z "$count" ] || [ "$count" -lt "$MIN_PENDING" ]; then
-              echo "FAIL: $label - WC_PENDING_E count too low:" \
-                "${count:-missing} (expected >= $MIN_PENDING)"
-              return 1
+            # Validate WC_PENDING_E count for sw mode only
+            # cryptocb mode uses callback pending which isn't tracked the same way
+            if [ "$ASYNC_MODE" = "sw" ]; then
+              local count
+              count=$(awk '/WC_PENDING_E count:/ {print $NF}' \
+                "/tmp/async_client_${label}.log")
+              if [ -z "$count" ] || [ "$count" -lt "$MIN_PENDING" ]; then
+                echo "FAIL: $label - WC_PENDING_E count too low:" \
+                  "${count:-missing} (expected >= $MIN_PENDING)"
+                return 1
+              fi
+              echo "PASS: $label (WC_PENDING_E: $count)"
+            else
+              echo "PASS: $label (cryptocb mode - connection successful)"
             fi
-            echo "PASS: $label (WC_PENDING_E: $count)"
             return 0
           }
 

examples/async/Makefile

@@ -5,6 +5,9 @@ RM ?= rm -f
 WOLFSSL_TOP ?= $(abspath ../..)
 OBJDIR ?= build
 
+# Async mode: "sw" (default) for WOLFSSL_ASYNC_CRYPT_SW, "cryptocb" for WOLF_CRYPTO_CB
+ASYNC_MODE ?= sw
+
 CFLAGS ?= -O0 -g
 CFLAGS += -I.
 CFLAGS += -I$(WOLFSSL_TOP)
@@ -14,6 +17,14 @@ CFLAGS += -Wall -Wextra -Wpedantic -Werror
 CFLAGS += -DWOLFSSL_USER_SETTINGS
 CFLAGS += -DHAVE_SYS_TIME_H
 CFLAGS += -DUSE_CERT_BUFFERS_256
+
+# Set async mode defines based on ASYNC_MODE
+ifeq ($(ASYNC_MODE),cryptocb)
+    CFLAGS += -DWOLF_CRYPTO_CB
+else
+    CFLAGS += -DWOLFSSL_ASYNC_CRYPT_SW
+endif
+
 CFLAGS += $(EXTRA_CFLAGS)
 
 LDFLAGS ?=

examples/async/README.md

@@ -14,8 +14,42 @@ Tested with:
 * `./configure --enable-asynccrypt --enable-pkcallbacks --enable-rsa --disable-ecc`
 * `./configure --enable-asynccrypt --enable-pkcallbacks --disable-rsa --enable-ecc`
 
+## Build Modes
+
+The async examples support two mutually exclusive async modes controlled via the
+`ASYNC_MODE` Makefile variable:
+
+### Software Async Mode (default)
+Uses `WOLFSSL_ASYNC_CRYPT_SW` with non-blocking ECC (`WC_ECC_NONBLOCK`):
 ```
 make -C examples/async
+# or explicitly:
+make -C examples/async ASYNC_MODE=sw
+```
+
+### Crypto Callback Mode
+Uses `WOLF_CRYPTO_CB` with the `AsyncTlsCryptoCb` callback that simulates hardware
+crypto delays by returning `WC_PENDING_E` for a configurable number of iterations:
+```
+make -C examples/async ASYNC_MODE=cryptocb
+```
+
+To adjust the simulated pending count (default is 2), define `TEST_PEND_COUNT`:
+```
+make -C examples/async ASYNC_MODE=cryptocb EXTRA_CFLAGS="-DTEST_PEND_COUNT=5"
+```
+
+To enable crypto callback debug output:
+```
+make -C examples/async ASYNC_MODE=cryptocb EXTRA_CFLAGS="-DDEBUG_CRYPTOCB"
+```
+
+**Note:** `WOLFSSL_ASYNC_CRYPT_SW` and `WOLF_CRYPTO_CB` are mutually exclusive in the
+async polling code (async.c uses `#elif`).
+
+## Running the Examples
+
+```
 ./examples/async/async_server --ecc
 ./examples/async/async_client --ecc 127.0.0.1 11111
 ./examples/async/async_client --x25519 ecc256.badssl.com 443

examples/async/async_client.c

@@ -52,6 +52,9 @@
 #include <wolfssl/ssl.h>
 #include <wolfssl/wolfio.h>
 #include <wolfssl/wolfcrypt/error-crypt.h>
+#ifdef WOLF_CRYPTO_CB
+#include <wolfssl/wolfcrypt/cryptocb.h>
+#endif
 #include <wolfssl/certs_test.h>
 #include "examples/async/async_tls.h"
 
@@ -233,6 +236,9 @@ int client_async_test(int argc, char** argv)
     int wouldblock_count = 0;
     int pending_count = 0;
 #endif
+#ifdef WOLF_CRYPTO_CB
+    AsyncTlsCryptoCbCtx cryptoCbCtx;
+#endif
 #ifdef WOLFSSL_STATIC_MEMORY
     static byte memory[300000];
     static byte memoryIO[34500];
@@ -279,6 +285,17 @@ int client_async_test(int argc, char** argv)
         goto out;
     }
 #endif
+#ifdef WOLF_CRYPTO_CB
+    /* Crypto callbacks require a valid devId. When no hardware async driver
+     * sets one (e.g. Cavium/Intel QA/SW), assign one explicitly. */
+    if (devId == INVALID_DEVID)
+        devId = 1;
+    XMEMSET(&cryptoCbCtx, 0, sizeof(cryptoCbCtx));
+    if (wc_CryptoCb_RegisterDevice(devId, AsyncTlsCryptoCb, &cryptoCbCtx) != 0) {
+        fprintf(stderr, "ERROR: wc_CryptoCb_RegisterDevice failed\n");
+        goto out;
+    }
+#endif
 
 #ifdef WOLFSSL_STATIC_MEMORY
     {
@@ -484,6 +501,8 @@ int client_async_test(int argc, char** argv)
             }
             continue;
         }
+        fprintf(stderr, "ERROR: wolfSSL_write failed: %d (%s)\n",
+            err, wolfSSL_ERR_reason_error_string(err));
         goto out;
     }
 
@@ -516,6 +535,8 @@ int client_async_test(int argc, char** argv)
             }
             continue;
         }
+        fprintf(stderr, "ERROR: wolfSSL_read failed: %d (%s)\n",
+            err, wolfSSL_ERR_reason_error_string(err));
         goto out;
     }
 
@@ -548,6 +569,9 @@ int client_async_test(int argc, char** argv)
     if (ctx != NULL) {
         wolfSSL_CTX_free(ctx);
     }
+#ifdef WOLF_CRYPTO_CB
+    wc_CryptoCb_UnRegisterDevice(devId);
+#endif
 #ifdef WOLFSSL_ASYNC_CRYPT
     if (devId != INVALID_DEVID) {
         wolfAsync_DevClose(&devId);

examples/async/async_server.c

@@ -57,6 +57,9 @@
 #include <wolfssl/ssl.h>
 #include <wolfssl/wolfio.h>
 #include <wolfssl/wolfcrypt/error-crypt.h>
+#ifdef WOLF_CRYPTO_CB
+#include <wolfssl/wolfcrypt/cryptocb.h>
+#endif
 #include <wolfssl/certs_test.h>
 #include "examples/async/async_tls.h"
 
@@ -191,6 +194,9 @@ int server_async_test(int argc, char** argv)
     int wouldblock_count = 0;
     int pending_count = 0;
 #endif
+#ifdef WOLF_CRYPTO_CB
+    AsyncTlsCryptoCbCtx cryptoCbCtx;
+#endif
 #ifdef WOLFSSL_STATIC_MEMORY
     static byte memory[300000];
     static byte memoryIO[34500];
@@ -284,6 +290,17 @@ int server_async_test(int argc, char** argv)
         goto exit;
     }
 #endif
+#ifdef WOLF_CRYPTO_CB
+    /* Crypto callbacks require a valid devId. When no hardware async driver
+     * sets one (e.g. Cavium/Intel QA/SW), assign one explicitly. */
+    if (devId == INVALID_DEVID)
+        devId = 1;
+    XMEMSET(&cryptoCbCtx, 0, sizeof(cryptoCbCtx));
+    if (wc_CryptoCb_RegisterDevice(devId, AsyncTlsCryptoCb, &cryptoCbCtx) != 0) {
+        fprintf(stderr, "ERROR: wc_CryptoCb_RegisterDevice failed\n");
+        goto exit;
+    }
+#endif
 
     /* Create and initialize WOLFSSL_CTX */
 #ifdef WOLFSSL_STATIC_MEMORY
@@ -513,6 +530,8 @@ int server_async_test(int argc, char** argv)
                 }
                 continue;
             }
+            fprintf(stderr, "ERROR: wolfSSL_read failed: %d (%s)\n",
+                err, wolfSSL_ERR_reason_error_string(err));
             goto exit;
         }
 
@@ -613,6 +632,9 @@ int server_async_test(int argc, char** argv)
     }
     if (ctx)
         wolfSSL_CTX_free(ctx);
+#ifdef WOLF_CRYPTO_CB
+    wc_CryptoCb_UnRegisterDevice(devId);
+#endif
 #ifdef WOLFSSL_ASYNC_CRYPT
     if (devId != INVALID_DEVID) {
         wolfAsync_DevClose(&devId);

examples/async/async_tls.c

@@ -23,8 +23,10 @@
     #include <config.h>
 #endif
 
-#ifndef WOLFSSL_USER_SETTINGS
-#include <wolfssl/options.h>
+#ifdef WOLFSSL_USER_SETTINGS
+    #include "user_settings.h"
+#else
+    #include <wolfssl/options.h>
 #endif
 #include "examples/async/async_tls.h"
 #include <wolfssl/ssl.h>
@@ -176,12 +178,17 @@ int AsyncTlsCryptoCb(int devIdArg, wc_CryptoInfo* info, void* ctx)
 
     if (info->algo_type == WC_ALGO_TYPE_PK) {
 #ifdef WOLFSSL_ASYNC_CRYPT
-        /* Test pending response */
+        /* Simulate async pending for RSA and ECC signing operations.
+         * This matches a typical hardware crypto scenario (e.g., TPM) where
+         * only signing is offloaded to hardware. Keygen, verify, and ECDH
+         * are performed synchronously in software.
+         * Note: WOLFSSL_ASYNC_CRYPT + WOLF_CRYPTO_CB pending simulation
+         * requires operations whose TLS state machines properly handle retry
+         * via wolfSSL_AsyncPop. ECC keygen in TLSX_KeyShare_GenEccKey does
+         * not support this because the keygen call is inside the key
+         * allocation guard (kse->key == NULL) which is skipped on retry. */
         if (info->pk.type == WC_PK_TYPE_RSA ||
-            info->pk.type == WC_PK_TYPE_EC_KEYGEN ||
-            info->pk.type == WC_PK_TYPE_ECDSA_SIGN ||
-            info->pk.type == WC_PK_TYPE_ECDSA_VERIFY ||
-            info->pk.type == WC_PK_TYPE_ECDH)
+            info->pk.type == WC_PK_TYPE_ECDSA_SIGN)
         {
             if (myCtx->pendingCount++ < TEST_PEND_COUNT) return WC_PENDING_E;
             myCtx->pendingCount = 0;

examples/async/user_settings.h

@@ -45,13 +45,20 @@
 #define WC_X25519_NONBLOCK
 
 #define WOLFSSL_ASYNC_CRYPT
-#define WOLFSSL_ASYNC_CRYPT_SW
 #define WC_NO_ASYNC_THREADING
 #define HAVE_WOLF_BIGINT
 
+/* Async mode is controlled via Makefile ASYNC_MODE variable:
+ * - ASYNC_MODE=sw (default): WOLFSSL_ASYNC_CRYPT_SW for non-blocking ECC
+ * - ASYNC_MODE=cryptocb: WOLF_CRYPTO_CB for crypto callback simulation
+ * These are mutually exclusive in async.c polling code (#elif).
+ * See README.md for build instructions.
+ */
+
 #define HAVE_AESGCM
 
 #define WOLFSSL_SHA512
+#define WOLFSSL_HASH_FLAGS
 
 #define WOLFSSL_TLS13
 #define HAVE_HKDF

src/internal.c

@@ -41802,7 +41802,10 @@ int wolfSSL_AsyncPop(WOLFSSL* ssl, byte* state)
              * the completion is not detected in the poll like Intel QAT or
              * Nitrox */
             ret = wolfEventQueue_Remove(&ssl->ctx->event_queue, event);
-
+            /* Clear async device so stale pending state from
+             * wolfSSL_AsyncInit does not confuse subsequent operations */
+            XMEMSET(&asyncDev->event, 0, sizeof(WOLF_EVENT));
+            ssl->asyncDev = NULL;
         }
     #endif
     }

wolfcrypt/src/async.c

@@ -705,7 +705,11 @@ int wolfAsync_EventQueuePoll(WOLF_EVENT_QUEUE* queue, void* context_filter,
                             event->ret = wolfAsync_DoSw(asyncDev);
                         }
                 #elif defined(WOLF_CRYPTO_CB) || defined(HAVE_PK_CALLBACKS)
-                    /* Use crypto or PK callbacks */
+                    /* Crypto/PK callbacks manage their own retry state.
+                     * Leave event->ret as WC_PENDING_E so that
+                     * wolfSSL_AsyncPop can detect the pending state and
+                     * remove the event, allowing the operation to be
+                     * retried with a fresh callback invocation. */
 
                 #else
                     #warning No async crypt device defined!

wolfcrypt/src/cryptocb.c

@@ -219,7 +219,7 @@ void wc_CryptoCb_InfoString(wc_CryptoInfo* info)
         printf("Crypto CB: %s %s (%d) (%p ctx)\n",
             GetAlgoTypeStr(info->algo_type),
             GetCipherTypeStr(info->cipher.type),
-            info->cipher.type, info->cipher.ctx);
+            info->cipher.type, (void*)info->cipher.ctx);
     }
 #endif /* !NO_AES || !NO_DES3 */
 #if !defined(NO_SHA) || !defined(NO_SHA256) || \
@@ -228,7 +228,7 @@ void wc_CryptoCb_InfoString(wc_CryptoInfo* info)
         printf("Crypto CB: %s %s (%d) (%p ctx) %s\n",
             GetAlgoTypeStr(info->algo_type),
             GetHashTypeStr(info->hash.type),
-            info->hash.type, info->hash.ctx,
+            info->hash.type, (void*)info->hash.ctx,
             (info->hash.in != NULL) ? "Update" : "Final");
     }
 #endif
@@ -237,7 +237,7 @@ void wc_CryptoCb_InfoString(wc_CryptoInfo* info)
         printf("Crypto CB: %s %s (%d) (%p ctx) %s\n",
             GetAlgoTypeStr(info->algo_type),
             GetHashTypeStr(info->hmac.macType),
-            info->hmac.macType, info->hmac.hmac,
+            info->hmac.macType, (void*)info->hmac.hmac,
             (info->hmac.in != NULL) ? "Update" : "Final");
     }
 #endif
@@ -246,7 +246,7 @@ void wc_CryptoCb_InfoString(wc_CryptoInfo* info)
         printf("Crypto CB: %s %s (%d) (%p ctx) %s %s %s\n",
             GetAlgoTypeStr(info->algo_type),
             GetCmacTypeStr(info->cmac.type),
-            info->cmac.type, info->cmac.cmac,
+            info->cmac.type, (void*)info->cmac.cmac,
             (info->cmac.key != NULL) ? "Init " : "",
             (info->cmac.in != NULL) ? "Update " : "",
             (info->cmac.out != NULL) ? "Final" : "");