ipc: userspace: fix IPC serialization with multiple cores

Fix a race in IPC serialization with multi-core. One sequence
observed:

- MOD_SET_DX IPC to power up core 1
- IPC reply to host
- CREATE_PIPELINE IPC (routed via core 0 to core 1)
- core 1 IPC thread starts, signals ipc_user->sem semaphore
- core 0 does NOT wait for thread as ipc_user->init_needed is set late
	- ipc_user->sem signal for thread start is handled as
	  indication that IPC is handled (this is wrong)
- IPC reply to host (before CREATE_PIPELINE is handled)
- INIT_INSTANCE IPC (routed via core 0 to core 1)
	- core 0 sees init_needed, but it is already signaled so execution
	  continues
	-> DSP panic as IPC mailbox is modified while still in use

Using a semaphore is not ideal to synchronize with the secondary core
IPC threads. Proper execution requires the host to send the correct IPC
sequence (e.g. first to send MOD_SET_DX for core x, and then follow-up
with IPC messages for same core x). Additionally the start-up is
different for first power-up of a secondary core (context is initialized
in memory), and subsequent power-ups (IPC thread is never terminated and
restarted, it just resumes when core is powered up again).

To handle all cases, replace the sem based synchronization with a simple
per-core bitmask to indicate whether IPC thread has been created.
Before forwarding IPC messages from ipc_user_forward_cmd(), use the bitmask
to check core status, and sleep if necessary.

This leaves ipc_user->sem dedicated to signal completion of IPC handling
in a user thread, and this will no longer get mixed with IPC thread boot
signaling.

Signed-off-by: Kai Vehmanen <kai.vehmanen@linux.intel.com>

Author: kv2019i

src/include/sof/ipc/common.h

@@ -82,7 +82,7 @@ struct ipc_user {
 	struct k_thread *audio_thread[IPC_DSP_CORE_COUNT];
 	/** @brief Original kernel driver pointer for restoring dev->drv after create */
 	const struct comp_driver *init_drv;
-	bool init_needed[IPC_DSP_CORE_COUNT];
+	atomic_t thread_ready;
 	/**
 	 * @brief User-accessible copy of comp_driver + tr_ctx for create().
 	 *

src/ipc/ipc-common.c

@@ -403,6 +403,8 @@ void ipc_schedule_process(struct ipc *ipc)
 #define IPC_USER_EVENT_CMD      BIT(0)
 #define IPC_USER_EVENT_STOP     BIT(1)
 
+#define IPC_USER_THREAD_TIMEOUT  100
+
 SOF_DEFINE_REG_UUID(sec_core_init);
 
 static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT,
@@ -437,13 +439,15 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core
 	pdata->ipc_msg_ext = extension;
 	pdata->ipc = ipc;
 
-	/*
-	 * Forwarding the first IPC to this core, wait for its userspace IPC
-	 * thread to start
-	 */
-	if (pdata->init_needed[core]) {
-		pdata->init_needed[core] = false;
-		k_sem_take(pdata->sem, K_FOREVER);
+	if (core != PLATFORM_PRIMARY_CORE_ID) {
+		/* Secondary cores: must ensure thread and matching pdata->event[core] is ready */
+		for (int cnt = 0; !atomic_test_bit(&pdata->thread_ready, core); ++cnt) {
+			k_msleep(1);
+			if (cnt == IPC_USER_THREAD_TIMEOUT) {
+				LOG_ERR("Unable to start core %u for IPC", core);
+				return -ETIME;
+			}
+		}
 	}
 
 	/* Prevent host completion until user thread finishes */
@@ -496,7 +500,12 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3)
 	__ASSERT(k_is_user_context(), "expected user context");
 
 	/* Signal startup complete — unblocks init waiting on semaphore */
-	k_sem_give(ipc_user->sem);
+	if (core == PLATFORM_PRIMARY_CORE_ID)
+		k_sem_give(ipc_user->sem);
+	else
+		/* allow IPCs to sent to this thread */
+		atomic_set_bit(&ipc_user->thread_ready, core);
+
 	LOG_INF("IPC user-space thread started");
 
 	for (;;) {
@@ -598,9 +607,6 @@ __cold int ipc_user_init_secondary(unsigned int core)
 	}
 
 	k_thread_access_grant(ipc_user->thread[core], ipc_user->audio_thread[core]);
-	ipc_user->init_needed[core] = true;
-
-	/* Wait for user thread startup — consumes the initial k_sem_give from thread */
 	return 0;
 }
 
@@ -628,6 +634,8 @@ __cold static void ipc_user_init(void)
 
 	assert_can_be_cold();
 
+	atomic_set(&ipc_user->thread_ready, 0);
+
 	ipc_user->sem = k_object_alloc(K_OBJ_SEM);
 	if (!ipc_user->sem) {
 		LOG_ERR("user IPC sem alloc failed");