FreeRDP
Loading...
Searching...
No Matches
audin_emscripten.c
1
18/*
19 * The main browser thread asks for the microphone (getUserMedia) and feeds it through an
20 * AudioWorklet. The worklet runs on the browser's audio thread: it converts every block to 16 bit
21 * PCM at the negotiated rate and channel count and appends it to a ring buffer in the shared wasm
22 * memory, then wakes the write position as a futex. A capture thread here waits on that futex and
23 * hands packets of FramesPerPacket frames to audin, which sends them as they are or encodes them
24 * (Opus, ...) for the server. Captured audio never waits for the busy main thread.
25 *
26 * Close() stops the worklet and the capture thread, then gives the ring buffer to the main thread,
27 * which frees it once the AudioContext is closed and the worklet can no longer write to it.
28 * Browsers only allow the microphone in a secure context after the user agreed, and only run an
29 * AudioContext after the user interacted with the page. Echo cancellation, noise suppression and
30 * automatic gain are on, as for a call; a page can set Module.freerdpAudinConstraints to other
31 * MediaTrackConstraints, e.g. to record music unprocessed. The worklet is
32 * audin_emscripten_worklet.js, served next to the executable or at Module.freerdpAudinWorkletUrl.
33 */
34
35#include <freerdp/config.h>
36
37#include <stdint.h>
38#include <stdlib.h>
39#include <string.h>
40
41#include <emscripten.h>
42#include <emscripten/proxying.h>
43#include <emscripten/threading.h>
44
45#include <winpr/crt.h>
46#include <winpr/synch.h>
47#include <winpr/thread.h>
48#include <winpr/wlog.h>
49
50#include <freerdp/types.h>
51#include <freerdp/addin.h>
52#include <freerdp/freerdp.h>
53#include <freerdp/codec/audio.h>
54#include <freerdp/client/audin.h>
55
56#include "audin_main.h"
57
58/* about 1.4 s at 48 kHz stereo; must be a power of two */
59#define AUDIN_WEB_RING_SAMPLES (1u << 17)
60
61typedef struct
62{
63 volatile int32_t active; /* 0: the worklet stops writing */
64 volatile uint32_t write_pos; /* samples written so far (wraps), woken as a futex */
65 int16_t ring[AUDIN_WEB_RING_SAMPLES];
66} audin_web_capture;
67
68typedef struct
69{
70 IAudinDevice iface;
71
72 AUDIO_FORMAT format;
73 UINT32 frames_per_packet;
74
75 AudinReceive receive;
76 void* user_data;
77
78 audin_web_capture* capture;
79 int capture_id;
80 int16_t* packet;
81 HANDLE thread;
82 volatile int32_t stop;
83
84 rdpContext* rdpcontext;
85 wLog* log;
86} AudinEmscriptenDevice;
87
88typedef struct
89{
90 int id;
91 audin_web_capture* capture;
92 int channels;
93 int rate;
94} audin_web_job;
95
96void audin_emscripten_release(void* capture);
97
98/* Called by the main thread when the AudioContext of a capture is closed. */
99EMSCRIPTEN_KEEPALIVE void audin_emscripten_release(void* capture)
100{
101 free(capture);
102}
103
104/* in audin_emscripten.js */
105extern void audin_web_start(int id, audin_web_capture* capture, int samples, int channels,
106 int rate);
107extern void audin_web_stop(int id, audin_web_capture* capture);
108
109static void start_on_main(void* arg)
110{
111 audin_web_job* job = arg;
112 audin_web_start(job->id, job->capture, (int)AUDIN_WEB_RING_SAMPLES, job->channels, job->rate);
113 free(job);
114}
115
116static void stop_on_main(void* arg)
117{
118 audin_web_job* job = arg;
119 audin_web_stop(job->id, job->capture);
120 free(job);
121}
122
123static BOOL run_on_main(void (*fn)(void*), audin_web_job* job)
124{
125 if (emscripten_is_main_browser_thread())
126 {
127 fn(job);
128 return TRUE;
129 }
130 return emscripten_proxy_async(emscripten_proxy_get_system_queue(),
131 emscripten_main_runtime_thread_id(), fn, job);
132}
133
134static DWORD WINAPI audin_emscripten_thread(LPVOID arg)
135{
136 AudinEmscriptenDevice* dev = arg;
137 audin_web_capture* capture = dev->capture;
138 const uint32_t packet = dev->frames_per_packet * dev->format.nChannels;
139 const size_t bytes = packet * sizeof(int16_t);
140 uint32_t read_pos = 0;
141
142 while (!__atomic_load_n(&dev->stop, __ATOMIC_ACQUIRE))
143 {
144 if (!__atomic_load_n(&capture->active, __ATOMIC_ACQUIRE))
145 {
146 if (!__atomic_load_n(&dev->stop, __ATOMIC_ACQUIRE))
147 WLog_Print(dev->log, WLOG_ERROR, "microphone capture failed, sending no audio");
148 break;
149 }
150
151 const uint32_t written = __atomic_load_n(&capture->write_pos, __ATOMIC_ACQUIRE);
152 const uint32_t available = written - read_pos;
153
154 if (available > AUDIN_WEB_RING_SAMPLES - packet)
155 {
156 /* the worklet overtook us: skip to the newest half of the ring */
157 WLog_Print(dev->log, WLOG_WARN,
158 "microphone capture overrun, dropping %" PRIu32 " samples",
159 available - AUDIN_WEB_RING_SAMPLES / 2);
160 read_pos = written - AUDIN_WEB_RING_SAMPLES / 2;
161 continue;
162 }
163
164 if (available < packet)
165 {
166 (void)emscripten_futex_wait(&capture->write_pos, written, 100.0);
167 continue;
168 }
169
170 for (uint32_t x = 0; x < packet; x++)
171 dev->packet[x] = capture->ring[(read_pos + x) & (AUDIN_WEB_RING_SAMPLES - 1)];
172 read_pos += packet;
173
174 const UINT error =
175 dev->receive(&dev->format, (const BYTE*)dev->packet, bytes, dev->user_data);
176 if (error)
177 {
178 if (dev->rdpcontext)
179 setChannelError(dev->rdpcontext, error,
180 "audin_emscripten_thread reported an error");
181 break;
182 }
183 }
184
185 ExitThread(0);
186 return 0;
187}
188
189static BOOL audin_emscripten_format_supported(WINPR_ATTR_UNUSED IAudinDevice* device,
190 const AUDIO_FORMAT* format)
191{
192 /* the worklet converts to any rate: 16 bit PCM, mono or stereo */
193 if (!format)
194 return FALSE;
195 return (format->wFormatTag == WAVE_FORMAT_PCM) && (format->wBitsPerSample == 16) &&
196 (format->nChannels >= 1) && (format->nChannels <= 2) &&
197 (format->nSamplesPerSec >= 8000) && (format->nSamplesPerSec <= 192000);
198}
199
200static UINT audin_emscripten_set_format(IAudinDevice* device, const AUDIO_FORMAT* format,
201 UINT32 FramesPerPacket)
202{
203 AudinEmscriptenDevice* dev = (AudinEmscriptenDevice*)device;
204
205 if (!dev || !audin_emscripten_format_supported(device, format))
206 return ERROR_INVALID_PARAMETER;
207
208 dev->format = *format;
209 dev->format.cbSize = 0;
210 dev->format.data = nullptr;
211
212 /* default to 20 ms, and keep a packet well inside the ring */
213 UINT32 frames = FramesPerPacket ? FramesPerPacket : format->nSamplesPerSec / 50;
214 const UINT32 max_frames = AUDIN_WEB_RING_SAMPLES / 4 / format->nChannels;
215 if (frames > max_frames)
216 frames = max_frames;
217 dev->frames_per_packet = frames;
218 return CHANNEL_RC_OK;
219}
220
221static UINT audin_emscripten_close(IAudinDevice* device)
222{
223 AudinEmscriptenDevice* dev = (AudinEmscriptenDevice*)device;
224
225 if (!dev)
226 return ERROR_INVALID_PARAMETER;
227 if (!dev->capture)
228 return CHANNEL_RC_OK;
229
230 __atomic_store_n(&dev->stop, 1, __ATOMIC_SEQ_CST);
231 __atomic_store_n(&dev->capture->active, 0, __ATOMIC_SEQ_CST);
232 (void)emscripten_futex_wake(&dev->capture->write_pos, INT32_MAX);
233
234 if (dev->thread)
235 {
236 (void)WaitForSingleObject(dev->thread, INFINITE);
237 (void)CloseHandle(dev->thread);
238 dev->thread = nullptr;
239 }
240
241 /* the main thread frees the ring once the worklet can no longer write to it */
242 audin_web_job* job = calloc(1, sizeof(audin_web_job));
243 if (job)
244 {
245 job->id = dev->capture_id;
246 job->capture = dev->capture;
247 }
248 if (!job || !run_on_main(stop_on_main, job))
249 {
250 WLog_Print(dev->log, WLOG_ERROR, "could not stop the microphone capture, leaking it");
251 free(job);
252 }
253
254 dev->capture = nullptr;
255 free(dev->packet);
256 dev->packet = nullptr;
257 dev->receive = nullptr;
258 dev->user_data = nullptr;
259 return CHANNEL_RC_OK;
260}
261
262static UINT audin_emscripten_open(IAudinDevice* device, AudinReceive receive, void* user_data)
263{
264 static int next_id = 0;
265 AudinEmscriptenDevice* dev = (AudinEmscriptenDevice*)device;
266
267 if (!dev || !receive || (dev->frames_per_packet == 0))
268 return ERROR_INVALID_PARAMETER;
269
270 UINT error = audin_emscripten_close(device);
271 if (error)
272 return error;
273
274 dev->packet = calloc(1ull * dev->frames_per_packet * dev->format.nChannels, sizeof(int16_t));
275 dev->capture = calloc(1, sizeof(audin_web_capture));
276 audin_web_job* job = calloc(1, sizeof(audin_web_job));
277 if (!dev->packet || !dev->capture || !job)
278 {
279 free(job);
280 error = CHANNEL_RC_NO_MEMORY;
281 goto fail;
282 }
283
284 dev->capture->active = 1;
285 dev->capture_id = __atomic_add_fetch(&next_id, 1, __ATOMIC_RELAXED);
286 dev->receive = receive;
287 dev->user_data = user_data;
288 __atomic_store_n(&dev->stop, 0, __ATOMIC_RELEASE);
289
290 dev->thread = CreateThread(nullptr, 0, audin_emscripten_thread, dev, 0, nullptr);
291 if (!dev->thread)
292 {
293 free(job);
294 error = ERROR_INTERNAL_ERROR;
295 goto fail;
296 }
297
298 job->id = dev->capture_id;
299 job->capture = dev->capture;
300 job->channels = dev->format.nChannels;
301 job->rate = (int)dev->format.nSamplesPerSec;
302 if (!run_on_main(start_on_main, job))
303 {
304 free(job);
305 (void)audin_emscripten_close(device);
306 return ERROR_INTERNAL_ERROR;
307 }
308 return CHANNEL_RC_OK;
309
310fail:
311 free(dev->packet);
312 dev->packet = nullptr;
313 free(dev->capture);
314 dev->capture = nullptr;
315 return error;
316}
317
318static UINT audin_emscripten_free(IAudinDevice* device)
319{
320 AudinEmscriptenDevice* dev = (AudinEmscriptenDevice*)device;
321
322 if (!dev)
323 return ERROR_INVALID_PARAMETER;
324
325 const UINT error = audin_emscripten_close(device);
326 free(dev);
327 return error;
328}
329
330FREERDP_ENTRY_POINT(UINT VCAPITYPE emscripten_freerdp_audin_client_subsystem_entry(
332{
333 AudinEmscriptenDevice* dev = calloc(1, sizeof(AudinEmscriptenDevice));
334 if (!dev)
335 return CHANNEL_RC_NO_MEMORY;
336
337 dev->log = WLog_Get(TAG);
338 dev->rdpcontext = pEntryPoints->rdpcontext;
339 dev->iface.Open = audin_emscripten_open;
340 dev->iface.FormatSupported = audin_emscripten_format_supported;
341 dev->iface.SetFormat = audin_emscripten_set_format;
342 dev->iface.Close = audin_emscripten_close;
343 dev->iface.Free = audin_emscripten_free;
344
345 const UINT error = pEntryPoints->pRegisterAudinDevice(pEntryPoints->plugin, &dev->iface);
346 if (error)
347 {
348 WLog_Print(dev->log, WLOG_ERROR, "RegisterAudinDevice failed with error %" PRIu32 "!",
349 error);
350 free(dev);
351 }
352 return error;
353}