From 9082c0892e9a351ab086a7a8b1ad294263b1cdd2 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 10:04:44 -0500 Subject: [PATCH 01/17] feat(bsp): microphone record/playback and tabbed example UIs for audio-capable boards Give the BSP components that have audio input a consistent way to exercise recording, playback, and volume control, and reorganize their examples into a multi-page UI so the widgets are no longer crammed onto one screen. Example UI (esp-box, t-deck, m5stack-tab5, esp32-p4-function-ev-board, smartpanlee-sc01-plus): - Reorganize each example into a tabbed (lv_tabview) UI so the IMU drawing, audio controls, and other widgets each get their own page. Drawing is confined to its own page, so swiping between tabs no longer smears points onto the other pages. - Wrapping tab labels (still readable when the display is rotated) and speaker icons on the volume buttons. Microphone support: - esp-box, t-deck, m5stack-tab5: on-screen record/playback GUI and a microphone-volume API, reading the ES7210 ADC. - esp32-p4-function-ev-board: record/playback GUI reading the ES8311 ADC. - m5stack-cardputer: microphone-volume API and function-key volume controls. - play_audio is now streaming-friendly (append what fits, report how many bytes were queued) so callers can stream buffers larger than the speaker stream buffer, and playback keeps that buffer topped up to avoid underrun static. On-screen volume labels stay in sync with the live BSP state. m5stack-tab5 (full-duplex ES8388 + ES7210) audio: - Fix the RX channel to share the TX standard-mode clock, enable the ES7210 high-pass filter, and drain RX from a dedicated microphone task (blocking read) rather than the shared playback task. On this single full-duplex controller an irregularly-drained RX ring perturbs the TX and both degrades the recording and pops the playback; the dedicated task mirrors the well-behaved esp-box path and yields clean capture. - The example applies DC removal and an RMS-normalized software makeup gain to each recording, so it plays back at an audible level from a low analog mic gain (the ES7210 front-end whines when the analog gain is driven hard). T-Deck ES7210 microphone (needed the most work): - Read the ES7210 as its fixed 4-slot TDM frame and keep the two populated microphones (MIC1 / MIC3); a narrower read misframes the codec. - Enable the ES7210 digital high-pass filter so the mic DC offset stops railing the ADC, and let the driver derive bclk_div so MCLK matches the codec coefficient table (a forced divider was slipping frames). - Run the example at 16 kHz so playback pitch is correct, and make the trackball center button optional (GPIO0 conflicts with the mic). - The mic is genuinely low-sensitivity and picks up random electrical impulse noise, so the example post-processes each recording: de-glitch (mark and interpolate), DC removal, and an RMS-normalized software makeup gain. This is the practical limit of the board's microphone hardware, documented in the example README. Example configuration (kept out of the BSP defaults): - Raise the LVGL task stacks, and on esp-box / t-deck the interrupt/touch task stack, through each example's sdkconfig.defaults so existing users of the components are unaffected; enable the end-of-stack watchpoint on those two boards and PSRAM where the recording buffers need it. All documented in the example READMEs. - Lower the m5stack-tab5 I/O-expander init log verbosity (INFO -> WARN). Co-Authored-By: Claude Opus 4.8 (1M context) --- components/esp-box/README.md | 6 +- components/esp-box/example/README.md | 28 ++ .../esp-box/example/main/esp_box_example.cpp | 191 ++++++++- components/esp-box/example/main/gui.cpp | 258 +++++++++--- components/esp-box/example/main/gui.hpp | 98 ++++- components/esp-box/example/sdkconfig.defaults | 14 + components/esp-box/include/esp-box.hpp | 64 ++- components/esp-box/src/audio.cpp | 157 ++++++- .../esp32-p4-function-ev-board/README.md | 6 +- .../example/README.md | 22 +- .../esp32_p4_function_ev_board_example.cpp | 141 ++++++- .../example/main/gui.cpp | 207 ++++++++-- .../example/main/gui.hpp | 75 +++- .../example/sdkconfig.defaults | 8 +- .../include/esp32-p4-function-ev-board.hpp | 64 ++- .../esp32-p4-function-ev-board/src/audio.cpp | 119 +++++- components/m5stack-cardputer/README.md | 5 +- .../m5stack-cardputer/example/README.md | 2 + .../m5stack-cardputer/example/main/gui.hpp | 2 + .../main/m5stack_cardputer_example.cpp | 15 + .../include/m5stack-cardputer.hpp | 16 + .../src/m5stack-cardputer.cpp | 14 +- .../m5stack-cardputer/src/microphone.cpp | 27 ++ components/m5stack-tab5/example/README.md | 23 +- components/m5stack-tab5/example/main/gui.cpp | 276 ++++++++++--- components/m5stack-tab5/example/main/gui.hpp | 98 ++++- .../example/main/m5stack_tab5_example.cpp | 214 +++++++++- .../m5stack-tab5/example/sdkconfig.defaults | 5 +- .../m5stack-tab5/include/m5stack-tab5.hpp | 36 +- components/m5stack-tab5/src/audio.cpp | 137 ++++--- components/m5stack-tab5/src/m5stack-tab5.cpp | 4 +- .../smartpanlee-sc01-plus/example/README.md | 18 +- .../example/main/gui.cpp | 152 +++++-- .../example/main/gui.hpp | 49 ++- .../main/smartpanlee_sc01_plus_example.cpp | 16 +- .../example/sdkconfig.defaults | 5 +- .../include/smartpanlee-sc01-plus.hpp | 10 +- .../src/smartpanlee-sc01-plus.cpp | 12 +- components/t-deck/CMakeLists.txt | 2 +- components/t-deck/README.md | 3 + components/t-deck/example/README.md | 32 ++ components/t-deck/example/main/gui.cpp | 214 ++++++++-- components/t-deck/example/main/gui.hpp | 86 +++- .../t-deck/example/main/t_deck_example.cpp | 385 +++++++++++++++++- components/t-deck/example/sdkconfig.defaults | 14 + components/t-deck/include/t-deck.hpp | 82 +++- components/t-deck/src/audio.cpp | 219 +++++++++- components/t-deck/src/t-deck.cpp | 15 +- 48 files changed, 3227 insertions(+), 419 deletions(-) mode change 100755 => 100644 components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp diff --git a/components/esp-box/README.md b/components/esp-box/README.md index c8c15ff21..6aa5d7208 100644 --- a/components/esp-box/README.md +++ b/components/esp-box/README.md @@ -8,7 +8,11 @@ and expansion headers. The `espp::EspBox` component provides a singleton hardware abstraction for initializing the touch, display, and audio subsystems, as well as automatically -determining which version of the Box it's running on. +determining which version of the Box it's running on. The audio subsystem runs +full duplex: `play_audio()` streams 16-bit stereo out through the ES8311 codec +and speaker, while `initialize_microphone()` delivers 16-bit stereo recordings +from the dual microphone array (through the ES7210 ADC, with adjustable gain +via `microphone_volume()`) at the speaker's sample rate. ## Example diff --git a/components/esp-box/example/README.md b/components/esp-box/example/README.md index dba524ce9..7bb168823 100644 --- a/components/esp-box/example/README.md +++ b/components/esp-box/example/README.md @@ -9,6 +9,16 @@ state and each time you touch the scren it 1) uses LVGL to draw a circle where you touch, and 2) play a click sound (wav file bundled with the firmware). If you press the home button on the display, it will clear the circles. +The on-screen audio row (bottom-left) exercises the microphones and speaker in +full duplex: the record button records from the dual microphone array (ES7210) +into a PSRAM-preferred buffer, the play button streams the recording back +through the speaker, and the remaining buttons adjust the speaker and +microphone volumes (shown in the label above the row). The measured effective +capture rate is logged when a recording stops. Because the board's +speaker is mono, the recorded stereo (mic 1 / mic 2) is downmixed to both +channels before playback, and the per-channel peak amplitude is logged so +you can tell a silent capture (peak near 0) from a playback issue. + https://github.com/user-attachments/assets/6fc9ce02-fdae-4f36-9ee7-3a6c347ca1c4 https://github.com/esp-cpp/espp/assets/213467/d5379983-9bc2-4d56-a9fc-9e37f54af15e @@ -17,6 +27,24 @@ https://github.com/esp-cpp/espp/assets/213467/d5379983-9bc2-4d56-a9fc-9e37f54af1 ![image](https://github.com/esp-cpp/espp/assets/213467/c7216cfd-330d-4610-baf2-30001c98ff42) ![image](https://github.com/user-attachments/assets/6d9901f1-f4fe-433a-b6d5-c8c82abd14b7) +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_ESP_BOX_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. + +The example also enables `CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK`, which +turns any future task-stack overflow into an immediate, clearly-named panic +instead of silent heap corruption. + ## How to use example ### Hardware Required diff --git a/components/esp-box/example/main/esp_box_example.cpp b/components/esp-box/example/main/esp_box_example.cpp index 4d6a2a099..69aaffa07 100644 --- a/components/esp-box/example/main/esp_box_example.cpp +++ b/components/esp-box/example/main/esp_box_example.cpp @@ -1,9 +1,16 @@ +#include +#include #include #include +#include +#include #include #include #include +#include +#include + #include "esp-box.hpp" #include "gui.hpp" @@ -17,6 +24,21 @@ static std::vector audio_bytes; static bool load_audio(size_t &out_size, size_t &out_sample_rate); static void play_click(espp::EspBox &box); +// Audio recording state (written by the microphone callback, read/controlled +// from the GUI button callbacks and main loop). The recorded data is 16-bit +// interleaved stereo at the current audio sample rate. +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 2; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static int64_t recording_start_us = 0; +static int64_t recording_last_us = 0; + extern "C" void app_main(void) { espp::Logger logger({.tag = "ESP BOX Example", .level = espp::Logger::Verbosity::INFO}); logger.info("Starting example!"); @@ -98,8 +120,9 @@ extern "C" void app_main(void) { // directly. static Gui gui({.log_level = espp::Logger::Verbosity::INFO}); static const std::string instructions = - fmt::format("\n\n\n\nTouch the screen!\nPress the home button or the {} button to clear " - "circles.\nPress the {} button to rotate the display.", + fmt::format("Touch the screen to draw!\nPress the home button or the {} button to clear " + "circles.\nPress the {} button to rotate the display.\nThe IMU and Audio " + "tabs show the other subsystems.", LV_SYMBOL_TRASH, LV_SYMBOL_REFRESH); gui.set_label_text(instructions); @@ -119,13 +142,18 @@ extern "C" void app_main(void) { if (touchpad_data.btn_state) { gui.clear_circles(); } - // if there is a touch point, draw a circle and play a click sound - if (touchpad_data.num_touch_points > 0) { + // if there is a touch point on the Draw tab, draw a circle and play a + // click sound (touches on the other tabs go to their widgets) + if (touchpad_data.num_touch_points > 0 && gui.draw_page_active()) { play_click(box); gui.draw_circle(touchpad_data.x, touchpad_data.y, 10); } } }; + // NOTE: this example raises the BSP interrupt-task stack size via + // sdkconfig.defaults (CONFIG_ESP_BOX_INTERRUPT_STACK_SIZE=8192); the + // touch controller is read from that task and its error-logging path + // needs more than the 4 KB BSP default. See the example README. if (!box.initialize_touch(touch_callback)) { logger.error("Failed to initialize touchpad!"); return; @@ -150,6 +178,95 @@ extern "C" void app_main(void) { // set the display brightness to be 75% box.brightness(75.0f); + // Initialize the microphones (the ES7210 shares the I2S bus with the + // speaker in full duplex, so recording runs at the speaker's sample rate) + // and buffer the recorded stereo frames; the recording auto-stops when the + // buffer is full and the main loop notices and updates the GUI + auto mic_callback = [](const uint8_t *data, size_t num_bytes) { + if (!recording) { + return; + } + size_t offset = recording_len; + size_t to_copy = std::min(num_bytes, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = esp_timer_get_time(); + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + recording = false; + } + }; + bool have_mic = box.initialize_microphone(mic_callback); + if (!have_mic) { + gui.set_audio_status("Mic unavailable (see log)"); + } + if (have_mic) { + // start at a modest microphone gain: the BSP default is fairly hot and + // can clip on close / loud sound. Nudge it up with the mic + button if + // recordings are too quiet. + box.microphone_volume(40.0f); + // allocate the recording buffer (16-bit interleaved stereo at the + // current sample rate): prefer PSRAM, fall back to a couple of seconds + // in internal RAM + size_t bytes_per_second = box.audio_sample_rate() * 2 * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + logger.warn("Could not allocate a recording buffer; recording disabled"); + gui.set_audio_status("No recording buffer"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz stereo)", recording_capacity / 1024, + recording_capacity / bytes_per_second, box.audio_sample_rate()); + } + } else { + logger.warn("Could not initialize the microphone!"); + } + + // The record button toggles recording; the play button toggles playback of + // the recording (streamed to the speaker by the main loop) + gui.set_record_callback([&]() { + if (!have_mic || recording_capacity == 0) { + logger.warn("Recording unavailable (no microphone / no buffer)"); + gui.set_audio_status("Mic unavailable (see log)"); + return; + } + if (recording) { + recording = false; // the main loop notices and logs the summary + } else { + playing = false; + gui.set_play_active(false); + recording_len = 0; + recording_start_us = esp_timer_get_time(); + recording_last_us = recording_start_us; + recording = true; + gui.set_record_active(true); + gui.set_audio_status("Recording..."); + } + }); + gui.set_play_callback([&]() { + if (playing) { + playing = false; + gui.set_play_active(false); + gui.set_audio_status("Playback stopped"); + } else if (recording_len > 0) { + recording = false; + playing = true; + gui.set_play_active(true); + gui.set_audio_status("Playing..."); + } else { + logger.info("Nothing recorded yet; press the record button first"); + gui.set_audio_status("Nothing recorded yet"); + } + }); + // make a task to read out the IMU data and update the GUI with it espp::Task imu_task( {.callback = [&](std::mutex &m, std::condition_variable &cv) -> bool { @@ -185,7 +302,7 @@ extern "C" void app_main(void) { gravity_vector.y = -gravity_vector.y; } - std::string text = fmt::format("{}\n\n\n\n\n", instructions); + std::string text; text += fmt::format("Accel: {:02.2f} {:02.2f} {:02.2f}\n", accel.x, accel.y, accel.z); text += fmt::format("Gyro: {:03.2f} {:03.2f} {:03.2f}\n", espp::deg_to_rad(gyro.x), espp::deg_to_rad(gyro.y), espp::deg_to_rad(gyro.z)); @@ -207,7 +324,7 @@ extern "C" void app_main(void) { // update the GUI with the new data; the Gui handles remapping the // vectors for the current display rotation - gui.set_label_text(text); + gui.set_imu_text(text); gui.set_kalman_down(gravity_vector.x, gravity_vector.y); gui.set_madgwick_down(vx, vy); @@ -221,9 +338,67 @@ extern "C" void app_main(void) { }}); imu_task.start(); - // loop forever + // Main loop: stream any active playback to the speaker and notice when a + // recording stops (either button press or the buffer filling up) + size_t play_offset = 0; + bool was_recording = false; while (true) { - std::this_thread::sleep_for(1s); + // feed the active playback in chunks, advancing by however much the + // speaker's stream buffer accepted + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_play_active(false); + gui.set_audio_status("Playback done"); + logger.info("Playback done"); + } else { + play_offset += box.play_audio(recording_buffer + play_offset, + std::min(len - play_offset, 16384)); + } + } else { + play_offset = 0; + } + // notice when the recording stopped (button press or buffer full) + bool now_recording = recording; + if (was_recording && !now_recording) { + gui.set_record_active(false); + gui.set_audio_status(fmt::format("Recorded {:.1f}s ({} plays)", + static_cast(recording_len) / + (box.audio_sample_rate() * 2 * sizeof(int16_t)), + LV_SYMBOL_PLAY)); + // report the measured capture rate: stereo frames recorded over the + // wall clock they took to arrive should match the nominal sample rate + size_t num_frames = recording_len / (2 * sizeof(int16_t)); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_frames / elapsed_s : 0.0f; + // Post-process the recording for playback on the box's MONO speaker: the + // ES7210 records 16-bit interleaved stereo (mic 1 -> left slot, mic 2 -> + // right), but the speaker only plays one I2S slot, so audio captured on + // the other slot would be inaudible. Downmix each L/R frame to the + // average and write it to BOTH slots so the mono speaker always plays + // it. Also report the peak amplitude per channel: a peak near 0 means + // the microphones captured silence (a codec / wiring problem), while a + // healthy peak means capture is fine. + auto *samples = reinterpret_cast(recording_buffer); + int16_t peak_left = 0, peak_right = 0; + for (size_t i = 0; i < num_frames; i++) { + int16_t l = samples[2 * i]; + int16_t r = samples[2 * i + 1]; + peak_left = std::max(peak_left, static_cast(std::abs(l))); + peak_right = std::max(peak_right, static_cast(std::abs(r))); + int16_t mono = static_cast((static_cast(l) + r) / 2); + samples[2 * i] = mono; + samples[2 * i + 1] = mono; + } + logger.info("Recorded {} frames in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal); " + "peak L={} R={} (of 32767 - near 0 means the mics captured silence)", + num_frames, elapsed_s, effective_hz, box.audio_sample_rate(), peak_left, + peak_right); + } + was_recording = now_recording; + std::this_thread::sleep_for(50ms); } //! [esp box example] } diff --git a/components/esp-box/example/main/gui.cpp b/components/esp-box/example/main/gui.cpp index f2c07468d..e477281c7 100644 --- a/components/esp-box/example/main/gui.cpp +++ b/components/esp-box/example/main/gui.cpp @@ -1,18 +1,15 @@ +#include #include #include "gui.hpp" void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); - init_label(); - init_gravity_lines(); - init_buttons(); + init_tabview(); + init_draw_tab(); + init_imu_tab(); + init_audio_tab(); init_circle_layer(); - // disable scrolling on the screen (so that it doesn't behave weirdly when - // rotated and drawing with your finger) - lv_obj_set_scrollbar_mode(lv_screen_active(), LV_SCROLLBAR_MODE_OFF); - lv_obj_clear_flag(lv_screen_active(), LV_OBJ_FLAG_SCROLLABLE); } void Gui::deinit_ui() { @@ -20,31 +17,67 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &box = espp::EspBox::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, box.lcd_width(), box.lcd_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(0, 0, 0), 0); +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); + imu_tab_ = lv_tabview_add_tab(tabview_, "IMU"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } -void Gui::init_label() { - label_ = lv_label_create(lv_screen_active()); +void Gui::init_draw_tab() { + // instructions on the left, with the rotate / clear buttons on the right + label_ = lv_label_create(draw_tab_); + lv_label_set_long_mode(label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(label_, lv_pct(70)); lv_label_set_text(label_, ""); lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 0, 0); lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_LEFT, 0); + + rotate_button_ = lv_btn_create(draw_tab_); + lv_obj_set_size(rotate_button_, 50, 50); + lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, 0, 0); + lv_obj_t *rotate_label = lv_label_create(rotate_button_); + lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); + lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); + + clear_button_ = lv_btn_create(draw_tab_); + lv_obj_set_size(clear_button_, 50, 50); + lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 60); + lv_obj_t *clear_label = lv_label_create(clear_button_); + lv_label_set_text(clear_label, LV_SYMBOL_TRASH); + lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); } -void Gui::init_gravity_lines() { - auto &box = espp::EspBox::get(); +void Gui::init_imu_tab() { + imu_label_ = lv_label_create(imu_tab_); + lv_label_set_long_mode(imu_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(imu_label_, lv_pct(100)); + lv_label_set_text(imu_label_, "IMU initializing..."); + lv_obj_align(imu_label_, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_set_style_text_align(imu_label_, LV_TEXT_ALIGN_LEFT, 0); lv_style_init(&kalman_line_style_); lv_style_set_line_width(&kalman_line_style_, 8); lv_style_set_line_color(&kalman_line_style_, lv_palette_main(LV_PALETTE_BLUE)); lv_style_set_line_rounded(&kalman_line_style_, true); - kalman_line_ = lv_line_create(lv_screen_active()); + kalman_line_ = lv_line_create(imu_tab_); kalman_line_points_[0] = {0, 0}; - kalman_line_points_[1] = {box.lcd_width(), box.lcd_height()}; + kalman_line_points_[1] = {0, 0}; lv_line_set_points(kalman_line_, kalman_line_points_, 2); lv_obj_add_style(kalman_line_, &kalman_line_style_, 0); @@ -53,46 +86,81 @@ void Gui::init_gravity_lines() { lv_style_set_line_color(&madgwick_line_style_, lv_palette_main(LV_PALETTE_RED)); lv_style_set_line_rounded(&madgwick_line_style_, true); - madgwick_line_ = lv_line_create(lv_screen_active()); + madgwick_line_ = lv_line_create(imu_tab_); madgwick_line_points_[0] = {0, 0}; - madgwick_line_points_[1] = {box.lcd_width(), box.lcd_height()}; + madgwick_line_points_[1] = {0, 0}; lv_line_set_points(madgwick_line_, madgwick_line_points_, 2); lv_obj_add_style(madgwick_line_, &madgwick_line_style_, 0); } -void Gui::init_buttons() { - // a button in the top left which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(rotate_button_, 50, 50); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_LEFT, 0, 0); - lv_obj_t *rotate_label = lv_label_create(rotate_button_); - lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); - lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); +void Gui::init_audio_tab() { + // a column: status line, volume line, then the buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 10, 0); - // a button in the top right which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(clear_button_, 50, 50); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 0); - lv_obj_t *clear_label = lv_label_create(clear_button_); - lv_label_set_text(clear_label, LV_SYMBOL_TRASH); - lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); + audio_status_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_status_label_, lv_pct(100)); + lv_label_set_text(audio_status_label_, "Idle"); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 10, 0); + lv_obj_set_style_pad_row(row, 10, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&record_button_, LV_SYMBOL_AUDIO}, {&play_button_, LV_SYMBOL_PLAY}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + {&mic_down_button_, LV_SYMBOL_MINUS}, {&mic_up_button_, LV_SYMBOL_PLUS}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 44, 44); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } + // remember the record / play button labels so set_record_active / + // set_play_active can swap their symbols + record_button_label_ = lv_obj_get_child(record_button_, 0); + play_button_label_ = lv_obj_get_child(play_button_, 0); + // color the volume buttons so the speaker and microphone pairs are + // distinguishable from each other + lv_obj_set_style_bg_color(mic_down_button_, lv_palette_main(LV_PALETTE_TEAL), 0); + lv_obj_set_style_bg_color(mic_up_button_, lv_palette_main(LV_PALETTE_TEAL), 0); } -void Gui::init_circle_layer() { +void Gui::update_audio_label() { auto &box = espp::EspBox::get(); + lv_label_set_text_fmt(audio_label_, + "Speaker %d%% (" LV_SYMBOL_VOLUME_MID "/" LV_SYMBOL_VOLUME_MAX + ")\nMic %d%% (teal " LV_SYMBOL_MINUS "/" LV_SYMBOL_PLUS ")", + static_cast(box.volume()), static_cast(box.microphone_volume())); +} + +void Gui::init_circle_layer() { + // a transparent, click-through overlay above the tabview which shows the + // touch trail (only populated while the Draw tab is active) circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, box.lcd_width(), box.lcd_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); lv_obj_set_style_bg_opa(circle_layer_, LV_OPA_TRANSP, 0); - lv_obj_set_style_border_width(circle_layer_, 0, 0); - lv_obj_set_style_outline_width(circle_layer_, 0, 0); - lv_obj_set_style_shadow_width(circle_layer_, 0, 0); lv_obj_add_event_cb(circle_layer_, draw_circle_layer, LV_EVENT_DRAW_MAIN, this); lv_obj_move_foreground(circle_layer_); } @@ -101,6 +169,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -116,13 +188,59 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_PRESSED: gui->on_pressed(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_pressed(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &box = espp::EspBox::get(); + if (target == record_button_) { + logger_.info("Record button pressed"); + if (record_callback_) { + record_callback_(); + } + return; + } + if (target == play_button_) { + logger_.info("Play button pressed"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + box.volume(box.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", box.volume()); + update_audio_label(); + return; + } + if (target == mic_down_button_ || target == mic_up_button_) { + float delta = target == mic_down_button_ ? -10.0f : 10.0f; + box.microphone_volume(box.microphone_volume() + delta); + logger_.info("Microphone volume: {:.0f}%", box.microphone_volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button pressed"); next_rotation(); @@ -132,21 +250,53 @@ void Gui::on_pressed(lv_event_t *e) { } } +void Gui::set_record_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(record_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_AUDIO); + lv_obj_set_style_bg_color( + record_button_, active ? lv_palette_main(LV_PALETTE_RED) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_play_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(play_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_PLAY); + lv_obj_set_style_bg_color( + play_button_, active ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_audio_status(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(audio_status_label_, std::string(text).c_str()); +} + void Gui::set_label_text(std::string_view text) { std::lock_guard lock(mutex_); lv_label_set_text(label_, std::string(text).c_str()); } +void Gui::set_imu_text(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(imu_label_, std::string(text).c_str()); +} + +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::next_rotation() { std::lock_guard lock(mutex_); - auto &box = espp::EspBox::get(); clear_circles_impl(); auto rotation = lv_display_get_rotation(lv_display_get_default()); rotation = static_cast((static_cast(rotation) + 1) % 4); lv_display_set_rotation(lv_display_get_default(), rotation); // update the size of the screen-filling objects - lv_obj_set_size(background_, box.rotated_display_width(), box.rotated_display_height()); - lv_obj_set_size(circle_layer_, box.rotated_display_width(), box.rotated_display_height()); + int width = lv_display_get_horizontal_resolution(lv_display_get_default()); + int height = lv_display_get_vertical_resolution(lv_display_get_default()); + lv_obj_set_size(tabview_, width, height); + lv_obj_set_size(circle_layer_, width, height); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_invalidate(circle_layer_); } @@ -162,7 +312,6 @@ void Gui::set_madgwick_down(float vx, float vy) { } void Gui::set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, float vy) { - auto &box = espp::EspBox::get(); // remap the vector according to the current display rotation so that the // line always points toward physical "down" auto rotation = lv_display_get_rotation(lv_display_get_default()); @@ -176,9 +325,16 @@ void Gui::set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, fl std::swap(vx, vy); vy = -vy; } - // draw the line from the center of the screen toward "down" - int x0 = box.rotated_display_width() / 2; - int y0 = box.rotated_display_height() / 2; + // draw the line from the center of the IMU tab's content area toward + // "down" (fall back to the display size if the layout hasn't run yet) + int w = lv_obj_get_content_width(imu_tab_); + int h = lv_obj_get_content_height(imu_tab_); + if (w <= 0 || h <= 0) { + w = lv_display_get_horizontal_resolution(lv_display_get_default()); + h = lv_display_get_vertical_resolution(lv_display_get_default()) - TAB_BAR_HEIGHT; + } + int x0 = w / 2; + int y0 = h / 2; points[0] = {static_cast(x0), static_cast(y0)}; points[1] = {static_cast(x0 + 50 * vx), static_cast(y0 + 50 * vy)}; diff --git a/components/esp-box/example/main/gui.hpp b/components/esp-box/example/main/gui.hpp index b3a2cdffc..7009aad03 100644 --- a/components/esp-box/example/main/gui.hpp +++ b/components/esp-box/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -22,15 +23,22 @@ /// dispatched through a single static trampoline (event_callback) into /// member functions, keeping all UI logic inside the class. /// -/// For this example the Gui shows: -/// * A label with instructions and live IMU data -/// * Two lines showing the direction of gravity ("down") as computed by a -/// Kalman filter (blue) and a Madgwick filter (red) -/// * A button (top-left) which rotates the display through 0/90/180/270 -/// * A button (top-right) which clears the circles drawn on the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Draw" tab: instructions, plus the rotate and clear buttons. Touching +/// the screen while this tab is active draws circles (on a transparent +/// overlay) and plays a click. +/// * "IMU" tab: live IMU text and two lines showing the direction of +/// gravity ("down") as computed by a Kalman filter (blue) and a Madgwick +/// filter (red). +/// * "Audio" tab: record / play buttons (wired to the example via +/// callbacks) and speaker / microphone volume buttons (acting directly on +/// the BSP), with a label showing the current volumes. class Gui { public: + /// Callback invoked when the record / play buttons are pressed + using audio_button_callback_t = std::function; + /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -49,10 +57,19 @@ class Gui { deinit_ui(); } - /// Set the text of the main label. Thread-safe. + /// Set the instruction text shown on the Draw tab. Thread-safe. /// @param text The text to display void set_label_text(std::string_view text); + /// Set the live IMU text shown on the IMU tab. Thread-safe. + /// @param text The text to display + void set_imu_text(std::string_view text); + + /// Whether the Draw tab is currently active (used by the example to only + /// draw circles / play clicks for touches on that tab). Thread-safe. + /// @return True if the Draw tab is the active tab + bool draw_page_active(); + /// Draw a circle at the given screen coordinates, replacing the oldest /// circle if the maximum number are already visible. Thread-safe. /// @param x The x coordinate (screen space) @@ -81,8 +98,34 @@ class Gui { /// @param vy The y component of the gravity vector void set_madgwick_down(float vx, float vy); + /// Set the callback invoked when the record button is pressed + /// @param callback The callback to invoke + void set_record_callback(audio_button_callback_t callback) { + record_callback_ = std::move(callback); + } + + /// Set the callback invoked when the play button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + + /// Show whether a recording is in progress (turns the record button red + /// and changes its symbol to stop). Thread-safe. + /// @param active True while recording + void set_record_active(bool active); + + /// Show whether a playback is in progress (turns the play button green and + /// changes its symbol to stop). Thread-safe. + /// @param active True while playing + void set_play_active(bool active); + + /// Set the status line on the Audio tab (e.g. "Recording...", "Mic + /// unavailable"). Thread-safe. + /// @param text The text to display + void set_audio_status(std::string_view text); + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 40; struct Circle { int x{0}; @@ -95,14 +138,19 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); - void init_label(); - void init_gravity_lines(); - void init_buttons(); + void init_tabview(); + void init_draw_tab(); + void init_imu_tab(); + void init_audio_tab(); void init_circle_layer(); + // update the audio volume label from the BSP's current volumes; called + // with the mutex held + void update_audio_label(); + // update the given "down" line from a vector in the unrotated display - // frame, remapping for the current display rotation + // frame, remapping for the current display rotation; called with the + // mutex held void set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, float vy); // the LVGL update task: calls lv_task_handler() under the mutex @@ -112,6 +160,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_pressed(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -122,14 +171,31 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *draw_tab_{nullptr}; + lv_obj_t *imu_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *label_{nullptr}; + lv_obj_t *imu_label_{nullptr}; lv_obj_t *kalman_line_{nullptr}; lv_obj_t *madgwick_line_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *record_button_{nullptr}; + lv_obj_t *record_button_label_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *play_button_label_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *mic_down_button_{nullptr}; + lv_obj_t *mic_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t record_callback_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + lv_style_t kalman_line_style_; lv_style_t madgwick_line_style_; lv_point_precise_t kalman_line_points_[2]; @@ -140,7 +206,9 @@ class Gui { size_t visible_circle_count_{0}; espp::Task update_task_{{.callback = [this](auto &m, auto &cv) { return update(m, cv); }, - .task_config = {.name = "gui", .stack_size_bytes = 6 * 1024}}}; + // NOTE: rendering the tabview (nested containers + flex layout) uses + // noticeably more stack than a flat UI; 6 KB overflows + .task_config = {.name = "gui", .stack_size_bytes = 12 * 1024}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/esp-box/example/sdkconfig.defaults b/components/esp-box/example/sdkconfig.defaults index 152da7c1f..933ed1454 100644 --- a/components/esp-box/example/sdkconfig.defaults +++ b/components/esp-box/example/sdkconfig.defaults @@ -34,3 +34,17 @@ CONFIG_LV_USE_THEME_DEFAULT=y CONFIG_LV_THEME_DEFAULT_DARK=y CONFIG_LV_THEME_DEFAULT_GROW=y CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=30 + +# PSRAM (octal PSRAM on the WROOM-1 R8 / R16V modules); used for the audio +# recording buffer in the example +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y + +# catch task stack overflows immediately (debug watchpoint on the stack end) +# instead of letting them silently corrupt the heap +CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y +# The interrupt task (touch I2C) logs errors through the fmt color-print +# path, which needs more than the 4 KB BSP default; give it 8 KB here so a +# transient touch-bus error cannot overflow its stack. See the README. +CONFIG_ESP_BOX_INTERRUPT_STACK_SIZE=8192 diff --git a/components/esp-box/include/esp-box.hpp b/components/esp-box/include/esp-box.hpp index cee3e549c..0487cbfd1 100644 --- a/components/esp-box/include/esp-box.hpp +++ b/components/esp-box/include/esp-box.hpp @@ -323,13 +323,59 @@ class EspBox : public BaseComponent { float volume() const; /// Play audio - /// \param data The audio data to play - void play_audio(const std::vector &data); + /// \param data The audio data to play (16-bit signed interleaved stereo) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(const std::vector &data); /// Play audio - /// \param data The audio data to play + /// \param data The audio data to play (16-bit signed interleaved stereo) /// \param num_bytes The number of bytes to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(const uint8_t *data, uint32_t num_bytes); + + ///////////////////////////////////////////////////////////////////////////// + // Microphone + ///////////////////////////////////////////////////////////////////////////// + + /// Alias for the microphone callback, called with recorded audio data + using microphone_callback_t = std::function; + + /// Initialize the microphones (the dual microphone array through the + /// ES7210 ADC) and start delivering audio data to the provided callback + /// \param callback The callback to call with recorded audio data: 16-bit + /// signed interleaved stereo (microphone 1 on the left slot, + /// microphone 2 on the right) at audio_sample_rate() + /// \param task_config The configuration for the microphone task + /// \return true if the microphone was successfully initialized, false + /// otherwise + /// \note The sound subsystem must be initialized first (the ES7210 shares + /// the I2S bus with the ES8311 in full duplex, so the microphones + /// record at the speaker's sample rate) + /// \note The callback runs in the microphone task's context, so the task's + /// stack must be large enough for whatever the callback does with + /// the audio data + bool initialize_microphone(const microphone_callback_t &callback, + const espp::Task::BaseConfig &task_config = {.name = "microphone", + .stack_size_bytes = 4096, + .priority = 10, + .core_id = 1}); + + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100), mapped onto the + /// ES7210 analog microphone gain range (0 dB - +37.5 dB) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100) + float microphone_volume() const; ///////////////////////////////////////////////////////////////////////////// // IMU @@ -358,6 +404,7 @@ class EspBox : public BaseComponent { bool update_touch(); void update_volume_output(); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); void lcd_wait_lines(); // box 3: @@ -536,6 +583,15 @@ class EspBox : public BaseComponent { StreamBufferHandle_t audio_tx_stream; i2s_std_config_t audio_std_cfg; + // microphone + std::shared_ptr> es7210_i2c_device_; + std::atomic microphone_initialized_{false}; + microphone_callback_t microphone_callback_{nullptr}; + std::unique_ptr microphone_task_{nullptr}; + std::vector audio_rx_buffer; + // microphone volume (percent), mapped onto the ES7210 analog gain range + std::atomic mic_volume_{70.0f}; + // IMU std::shared_ptr> imu_i2c_device_; std::shared_ptr imu_; diff --git a/components/esp-box/src/audio.cpp b/components/esp-box/src/audio.cpp index 732f3874c..bba4aacab 100644 --- a/components/esp-box/src/audio.cpp +++ b/components/esp-box/src/audio.cpp @@ -1,3 +1,6 @@ +#include +#include + #include "esp-box.hpp" using namespace espp; @@ -6,6 +9,14 @@ using namespace espp; // Audio Functions // //////////////////////// +// Map a 0-100% microphone volume onto the ES7210's analog gain steps +// (GAIN_0DB .. GAIN_37_5DB) +static es7210_gain_value_t microphone_gain_from_volume(float volume) { + int step = static_cast(std::lround(volume / 100.0f * static_cast(GAIN_37_5DB))); + step = std::clamp(step, static_cast(GAIN_0DB), static_cast(GAIN_37_5DB)); + return static_cast(step); +} + bool EspBox::initialize_codec() { logger_.info("initializing codec"); @@ -156,7 +167,7 @@ void EspBox::mute(bool mute) { bool EspBox::is_muted() const { return mute_; } void EspBox::volume(float volume) { - volume_ = volume; + volume_ = std::clamp(volume, 0.0f, 100.0f); update_volume_output(); } @@ -183,9 +194,145 @@ void EspBox::audio_sample_rate(uint32_t sample_rate) { i2s_channel_enable(audio_tx_handle); } -void EspBox::play_audio(const std::vector &data) { play_audio(data.data(), data.size()); } +size_t EspBox::play_audio(const std::vector &data) { + return play_audio(data.data(), data.size()); +} + +size_t EspBox::play_audio(const uint8_t *data, uint32_t num_bytes) { + // don't block here: append what fits into the stream buffer and report how + // much was actually queued so callers can stream data larger than the + // buffer + return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); +} + +////////////////////////// +// Microphone Functions // +////////////////////////// + +bool EspBox::initialize_microphone(const microphone_callback_t &callback, + const espp::Task::BaseConfig &task_config) { + logger_.info("Initializing microphone"); + if (microphone_initialized_) { + logger_.warn("Microphone already initialized, not initializing again!"); + return false; + } + if (!sound_initialized_) { + logger_.error("The sound subsystem must be initialized first: the ES7210 shares the I2S bus " + "with the ES8311 in full duplex"); + return false; + } + if (!callback) { + logger_.error("A callback is required to receive the recorded audio data"); + return false; + } + microphone_callback_ = callback; + + // The RX channel was allocated alongside the TX channel in + // initialize_i2s(); in full-duplex mode it shares the TX BCLK/WS, so + // initialize it with the same standard-mode (16-bit stereo) config + ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_rx_handle, &audio_std_cfg)); + audio_rx_buffer.resize(calc_audio_buffer_size(audio_sample_rate())); + ESP_ERROR_CHECK(i2s_channel_enable(audio_rx_handle)); + + // Configure the ES7210 ADC (microphone 1 -> left slot, microphone 2 -> + // right slot) now that the I2S clocks are running. Probe for the chip + // first: the address depends on how its AD pins are strapped, so accept + // either 0x40 or 0x41 and report clearly if neither responds. + uint8_t es7210_address = 0; + for (uint8_t address : {0x40, 0x41}) { + if (internal_i2c_.probe_device(address)) { + es7210_address = address; + break; + } + } + if (es7210_address == 0) { + logger_.error("No ES7210 found on the internal I2C bus (probed 0x40 and 0x41); " + "cannot record from the microphones"); + i2s_channel_disable(audio_rx_handle); + return false; + } + logger_.info("Found ES7210 at {:#04x}", es7210_address); + std::error_code ec; + es7210_i2c_device_ = internal_i2c_.add_device( + { + .device_address = es7210_address, + .timeout_ms = static_cast(internal_i2c_.config().timeout_ms), + .scl_speed_hz = internal_i2c_.config().clk_speed, + .log_level = espp::Logger::Verbosity::WARN, + }, + ec); + if (!es7210_i2c_device_) { + logger_.error("Could not initialize ES7210 I2C device: {}", ec.message()); + i2s_channel_disable(audio_rx_handle); + return false; + } + set_es7210_write(espp::make_i2c_addressed_write(es7210_i2c_device_)); + set_es7210_read(espp::make_i2c_addressed_read_register(es7210_i2c_device_)); + + // only the two microphones are wired on the box (the other ES7210 inputs + // are unused), and they map onto the two I2S slots + es7210_mic_select(static_cast(ES7210_INPUT_MIC1 | ES7210_INPUT_MIC2)); + + audio_hal_codec_config_t es7210_cfg{}; + es7210_cfg.codec_mode = AUDIO_HAL_CODEC_MODE_ENCODE; + es7210_cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; + es7210_cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; + es7210_cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; + es7210_cfg.i2s_iface.samples = AUDIO_HAL_48K_SAMPLES; + if (es7210_adc_init(&es7210_cfg) != ESP_OK) { + logger_.error("Could not initialize the ES7210 codec"); + i2s_channel_disable(audio_rx_handle); + return false; + } + if (es7210_adc_config_i2s(AUDIO_HAL_CODEC_MODE_ENCODE, &es7210_cfg.i2s_iface) != ESP_OK) { + logger_.error("ES7210 I2S config failed"); + } + // Enable the ES7210 digital high-pass filter on the ADC channels to strip + // the microphone DC offset. The espp es7210 driver leaves these registers + // at their reset value (HPF off), so the DC offset is amplified by the + // analog gain and rails the ADC to full scale - recordings come out as a + // near-constant DC level that the AC-coupled speaker plays as a click then + // silence. These are the driver's documented "quick setup" HPF values. + std::error_code hpf_ec; + es7210_i2c_device_->write_register(0x22, std::vector{0x0a}, hpf_ec); // ADC1/2 HPF1 + es7210_i2c_device_->write_register(0x23, std::vector{0x2a}, hpf_ec); // ADC1/2 HPF2 + es7210_i2c_device_->write_register(0x20, std::vector{0x0a}, hpf_ec); // ADC3/4 HPF2 + es7210_i2c_device_->write_register(0x21, std::vector{0x2a}, hpf_ec); // ADC3/4 HPF1 + if (hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", hpf_ec.message()); + } + // apply the stored microphone volume (the driver's init leaves the analog + // gain at its 0 dB minimum, which records very quietly) + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + es7210_adc_ctrl_state(AUDIO_HAL_CODEC_MODE_ENCODE, AUDIO_HAL_CTRL_START); + + using namespace std::placeholders; + microphone_task_ = espp::Task::make_unique({ + .callback = std::bind(&EspBox::microphone_task_callback, this, _1, _2, _3), + .task_config = task_config, + }); + + microphone_initialized_ = true; + + return microphone_task_->start(); +} + +bool EspBox::microphone_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + size_t bytes_read = 0; + auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), + &bytes_read, portMAX_DELAY); + if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { + microphone_callback_(audio_rx_buffer.data(), bytes_read); + } + return false; // don't stop the task +} -void EspBox::play_audio(const uint8_t *data, uint32_t num_bytes) { - // don't block here - xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); +void EspBox::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (microphone_initialized_) { + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + } } + +float EspBox::microphone_volume() const { return mic_volume_; } diff --git a/components/esp32-p4-function-ev-board/README.md b/components/esp32-p4-function-ev-board/README.md index 55f30a1d6..8459f61ce 100644 --- a/components/esp32-p4-function-ev-board/README.md +++ b/components/esp32-p4-function-ev-board/README.md @@ -51,7 +51,11 @@ initializes and exposes the board's peripherals: - **GT911 capacitive multi-touch** — polled by default (the touch INT pin is not routed to the ESP32-P4 on this board), or **interrupt-driven** if you wire the INT pin from the LCD expansion header to a GPIO (see [Touch mode](#touch-mode-polling-vs-interrupt)). -- **ES8311 audio codec** (+ NS4150B speaker amplifier) over I2S for playback. +- **ES8311 audio codec** (+ NS4150B speaker amplifier) over I2S, full duplex: + 16-bit mono playback via `play_audio()` and recording from the onboard + analog microphone via `initialize_microphone()` (16-bit mono at the + speaker's sample rate, with adjustable analog gain via + `microphone_volume()`). - **10/100 Ethernet** (EMAC + IP101 RMII PHY) with DHCP. - **microSD card** (4-bit SDMMC, powered via the on-chip LDO). - **MIPI-CSI camera** (SC2336/OV5647) — pins/SCCB wired, capture pipeline is a diff --git a/components/esp32-p4-function-ev-board/example/README.md b/components/esp32-p4-function-ev-board/example/README.md index 384e2a24d..a0016e4da 100644 --- a/components/esp32-p4-function-ev-board/example/README.md +++ b/components/esp32-p4-function-ev-board/example/README.md @@ -15,7 +15,11 @@ It: card size, Ethernet IP, **RTPS publisher** status, and **system** info (free internal/PSRAM heap and uptime), - mounts the microSD card (if inserted), -- initializes the ES8311 audio codec (and loads an embedded `click.wav`), +- initializes the ES8311 audio codec (and loads an embedded `click.wav`); + the on-screen audio row (bottom-left) records from the onboard microphone + into a PSRAM-preferred buffer, plays the recording back through the + speaker, and adjusts the speaker / microphone volumes (the measured + effective capture rate is logged when a recording stops), - brings up Ethernet (IP101) with DHCP and, once it has an IP, starts an **RTPS participant** that publishes a counter on `espp/test/counter`, and - wires the BOOT button. @@ -25,6 +29,22 @@ It: > ESP32-P4 main board (`RST_LCD` → J1 GPIO27, `PWM` → J1 GPIO26) or the screen > stays black. See the component README. +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_ESP_P4_EV_BOARD_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. +- **Polled touch task stack (`CONFIG_ESP_P4_EV_BOARD_TOUCH_TASK_STACK_SIZE` = 8192).** Same reason: + this board polls the GT911 over I2C from its own task. + ## Configuration Use `idf.py menuconfig` → *ESP32-P4 Function EV Board Configuration* to select diff --git a/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp b/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp index daf674264..40e65fd6e 100644 --- a/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp +++ b/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -39,6 +40,21 @@ static std::vector audio_bytes; static bool load_audio(size_t &out_size, size_t &out_sample_rate); +// Audio recording state (written by the microphone callback, read/controlled +// from the GUI button callbacks and main loop). The recorded data is 16-bit +// mono at the audio sample rate. +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 2; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static int64_t recording_start_us = 0; +static int64_t recording_last_us = 0; + // Ping a target host a few times and log the result (uses the espp Ping helper). static void ping_target(espp::Logger &logger, const char *name, const std::string &ip) { if (ip.empty() || ip == "0.0.0.0") { @@ -187,6 +203,11 @@ extern "C" void app_main(void) { // touch-down or once the point has moved at least one radius, so a stationary // touch draws a single circle and a drag leaves a spaced trail. static constexpr int kCircleRadius = 10; + // NOTE: this example raises the BSP interrupt- and touch-task stack sizes + // via sdkconfig.defaults (CONFIG_ESP_P4_EV_BOARD_INTERRUPT_STACK_SIZE / + // _TOUCH_TASK_STACK_SIZE = 8192); the touch controller is read (polled) + // from those tasks and their error-logging path needs more than the 4 KB + // BSP default. See the example README. board.initialize_touch([&](const auto &data) { auto td = board.touchpad_convert(data); static Board::TouchpadData prev_td = {}; @@ -198,7 +219,7 @@ extern "C" void app_main(void) { if (new_touch && !audio_bytes.empty()) { board.play_audio(audio_bytes); // non-blocking, touch-down edge only } - if (new_touch) { + if (new_touch && gui.draw_page_active()) { gui.draw_circle(td.x, td.y, kCircleRadius); } } @@ -229,8 +250,89 @@ extern "C" void app_main(void) { if (have_audio) { logger.info("Loaded {} bytes of click audio @ {} Hz", wav_size, wav_sample_rate); } + + // Microphone: the ES8311 is full duplex, so the onboard microphone + // records at the speaker's sample rate. Buffer the recorded mono samples + // and auto-stop when the buffer is full (the main loop notices and + // updates the GUI). + auto mic_callback = [](const uint8_t *data, size_t num_bytes) { + if (!recording) { + return; + } + size_t offset = recording_len; + size_t to_copy = std::min(num_bytes, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = esp_timer_get_time(); + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + recording = false; + } + }; + if (board.initialize_microphone(mic_callback)) { + // allocate the recording buffer (16-bit mono at the current sample + // rate): prefer PSRAM, fall back to a couple of seconds in internal RAM + size_t bytes_per_second = board.audio_sample_rate() * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + logger.warn("Could not allocate a recording buffer; recording disabled"); + gui.set_audio_status("No recording buffer"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz mono)", recording_capacity / 1024, + recording_capacity / bytes_per_second, board.audio_sample_rate()); + } + } else { + logger.warn("Could not initialize the microphone!"); + gui.set_audio_status("Mic unavailable (see log)"); + } } + // The record button toggles recording; the play button toggles playback of + // the recording (streamed to the speaker by the main loop) + gui.set_record_callback([&]() { + if (recording_capacity == 0) { + logger.warn("Recording unavailable (no microphone / no buffer)"); + gui.set_audio_status("Mic unavailable (see log)"); + return; + } + if (recording) { + recording = false; // the main loop notices and logs the summary + } else { + playing = false; + gui.set_play_active(false); + recording_len = 0; + recording_start_us = esp_timer_get_time(); + recording_last_us = recording_start_us; + recording = true; + gui.set_record_active(true); + gui.set_audio_status("Recording..."); + } + }); + gui.set_play_callback([&]() { + if (playing) { + playing = false; + gui.set_play_active(false); + gui.set_audio_status("Playback stopped"); + } else if (recording_len > 0) { + recording = false; + playing = true; + gui.set_play_active(true); + gui.set_audio_status("Playing..."); + } else { + logger.info("Nothing recorded yet; press the record button first"); + gui.set_audio_status("Nothing recorded yet"); + } + }); + // Ethernet (IP101) — DHCP; the callback fires once an IP is acquired static std::atomic have_ip{false}; static std::string ip_str{"(no link)"}; @@ -380,6 +482,43 @@ extern "C" void app_main(void) { rtps_value = value; rtps_has_peers = published; + // Stream any active playback to the speaker in chunks, advancing by + // however much the stream buffer accepted + static size_t play_offset = 0; + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_play_active(false); + gui.set_audio_status("Playback done"); + logger.info("Playback done"); + } else { + play_offset += board.play_audio(recording_buffer + play_offset, + std::min(len - play_offset, 16384)); + } + } else { + play_offset = 0; + } + // Notice when the recording stopped (button press or buffer full) + static bool was_recording = false; + bool now_recording = recording; + if (was_recording && !now_recording) { + gui.set_record_active(false); + gui.set_audio_status(fmt::format("Recorded {:.1f}s ({} plays)", + static_cast(recording_len) / + (board.audio_sample_rate() * sizeof(int16_t)), + LV_SYMBOL_PLAY)); + // report the measured capture rate: samples recorded over the wall + // clock they took to arrive should match the nominal sample rate + size_t num_samples = recording_len / sizeof(int16_t); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_samples / elapsed_s : 0.0f; + logger.info("Recorded {} samples in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal)", + num_samples, elapsed_s, effective_hz, board.audio_sample_rate()); + } + was_recording = now_recording; + std::this_thread::sleep_for(loop_tick); } } diff --git a/components/esp32-p4-function-ev-board/example/main/gui.cpp b/components/esp32-p4-function-ev-board/example/main/gui.cpp index 2bf3c3b6f..4928eabc2 100644 --- a/components/esp32-p4-function-ev-board/example/main/gui.cpp +++ b/components/esp32-p4-function-ev-board/example/main/gui.cpp @@ -1,17 +1,35 @@ +#include +#include + #include "gui.hpp" using Board = espp::Esp32P4FunctionEvBoard; void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); + init_tabview(); init_labels(); init_buttons(); + init_audio_controls(); init_circle_layer(); - // disable scrolling on the screen (so that it doesn't behave weirdly when - // rotated and drawing with your finger) - lv_obj_set_scrollbar_mode(lv_screen_active(), LV_SCROLLBAR_MODE_OFF); - lv_obj_clear_flag(lv_screen_active(), LV_OBJ_FLAG_SCROLLABLE); +} + +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + status_tab_ = lv_tabview_add_tab(tabview_, "Status"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } void Gui::deinit_ui() { @@ -19,53 +37,127 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &board = Board::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, board.display_width(), board.display_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(0, 0, 0), 0); -} - void Gui::init_labels() { - // a title label at the top of the screen - title_label_ = lv_label_create(lv_screen_active()); + // a title label at the top of the Status tab + title_label_ = lv_label_create(status_tab_); + lv_label_set_long_mode(title_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(title_label_, lv_pct(100)); lv_label_set_text(title_label_, "ESP32-P4 Function EV Board - touch to draw!"); - lv_obj_align(title_label_, LV_ALIGN_TOP_MID, 0, 8); + lv_obj_align(title_label_, LV_ALIGN_TOP_MID, 0, 0); // a status label showing the live subsystem state, updated via // set_status_text() - status_label_ = lv_label_create(lv_screen_active()); + status_label_ = lv_label_create(status_tab_); + lv_label_set_long_mode(status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(status_label_, lv_pct(100)); lv_label_set_text(status_label_, ""); lv_obj_set_style_text_align(status_label_, LV_TEXT_ALIGN_LEFT, 0); - lv_obj_align(status_label_, LV_ALIGN_TOP_LEFT, 8, 40); + lv_obj_align(status_label_, LV_ALIGN_TOP_LEFT, 0, 32); } void Gui::init_buttons() { - // a button in the top right which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); + // a button in the top right of the Status tab which rotates the display + // through 0/90/180/270 degrees + rotate_button_ = lv_btn_create(status_tab_); lv_obj_set_size(rotate_button_, 50, 50); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, -8, 8); + lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, 0, 0); lv_obj_t *rotate_label = lv_label_create(rotate_button_); lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_CLICKED, this); // a button next to it which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); + clear_button_ = lv_btn_create(status_tab_); lv_obj_set_size(clear_button_, 50, 50); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, -66, 8); + lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, -58, 0); lv_obj_t *clear_label = lv_label_create(clear_button_); lv_label_set_text(clear_label, LV_SYMBOL_TRASH); lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_CLICKED, this); } +void Gui::init_audio_controls() { + // the Audio tab: a column with a status line, the volume line, then the + // buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 12, 0); + + audio_status_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_status_label_, lv_pct(100)); + lv_label_set_text(audio_status_label_, "Idle"); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 12, 0); + lv_obj_set_style_pad_row(row, 12, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&record_button_, LV_SYMBOL_AUDIO}, {&play_button_, LV_SYMBOL_PLAY}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + {&mic_down_button_, LV_SYMBOL_MINUS}, {&mic_up_button_, LV_SYMBOL_PLUS}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 50, 50); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_CLICKED, this); + } + // remember the record / play button labels so set_record_active / + // set_play_active can swap their symbols + record_button_label_ = lv_obj_get_child(record_button_, 0); + play_button_label_ = lv_obj_get_child(play_button_, 0); + // color the volume buttons so the speaker and microphone pairs are + // distinguishable from each other + lv_obj_set_style_bg_color(mic_down_button_, lv_palette_main(LV_PALETTE_TEAL), 0); + lv_obj_set_style_bg_color(mic_up_button_, lv_palette_main(LV_PALETTE_TEAL), 0); +} + +void Gui::update_audio_label() { + auto &board = espp::Esp32P4FunctionEvBoard::get(); + lv_label_set_text_fmt(audio_label_, + "Speaker %d%% (" LV_SYMBOL_VOLUME_MID "/" LV_SYMBOL_VOLUME_MAX + ")\nMic %d%% (teal " LV_SYMBOL_MINUS "/" LV_SYMBOL_PLUS ")", + static_cast(board.volume()), + static_cast(board.microphone_volume())); +} + +void Gui::set_record_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(record_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_AUDIO); + lv_obj_set_style_bg_color( + record_button_, active ? lv_palette_main(LV_PALETTE_RED) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_play_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(play_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_PLAY); + lv_obj_set_style_bg_color( + play_button_, active ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + void Gui::init_circle_layer() { - auto &board = Board::get(); + // a transparent, click-through overlay above the tabview which shows the + // touch trail (only populated while the Status tab is active) circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, board.display_width(), board.display_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); @@ -81,6 +173,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -96,13 +192,59 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_CLICKED: gui->on_clicked(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_clicked(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &board = espp::Esp32P4FunctionEvBoard::get(); + if (target == record_button_) { + logger_.info("Record button clicked"); + if (record_callback_) { + record_callback_(); + } + return; + } + if (target == play_button_) { + logger_.info("Play button clicked"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + board.volume(board.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", board.volume()); + update_audio_label(); + return; + } + if (target == mic_down_button_ || target == mic_up_button_) { + float delta = target == mic_down_button_ ? -10.0f : 10.0f; + board.microphone_volume(board.microphone_volume() + delta); + logger_.info("Microphone volume: {:.0f}%", board.microphone_volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button clicked"); next_rotation(); @@ -117,6 +259,16 @@ void Gui::set_status_text(std::string_view text) { lv_label_set_text(status_label_, std::string(text).c_str()); } +void Gui::set_audio_status(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(audio_status_label_, std::string(text).c_str()); +} + +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::next_rotation() { std::lock_guard lock(mutex_); auto &board = Board::get(); @@ -125,14 +277,11 @@ void Gui::next_rotation() { rotation = static_cast((static_cast(rotation) + 1) % 4); lv_display_set_rotation(lv_display_get_default(), rotation); // update the size of the screen-filling objects - lv_obj_set_size(background_, board.rotated_display_width(), board.rotated_display_height()); + lv_obj_set_size(tabview_, board.rotated_display_width(), board.rotated_display_height()); lv_obj_set_size(circle_layer_, board.rotated_display_width(), board.rotated_display_height()); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_move_foreground(circle_layer_); lv_obj_invalidate(circle_layer_); - // re-align the labels to the (now reoriented) screen edges - lv_obj_align(title_label_, LV_ALIGN_TOP_MID, 0, 8); - lv_obj_align(status_label_, LV_ALIGN_TOP_LEFT, 8, 40); } void Gui::draw_circle(int x, int y, int radius) { diff --git a/components/esp32-p4-function-ev-board/example/main/gui.hpp b/components/esp32-p4-function-ev-board/example/main/gui.hpp index 8f6836b6f..da8cc24fa 100644 --- a/components/esp32-p4-function-ev-board/example/main/gui.hpp +++ b/components/esp32-p4-function-ev-board/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -26,11 +27,19 @@ /// * A title label at the top of the screen /// * A status label showing live subsystem state (panel, touch, SD, Ethernet, /// RTPS, and system memory/uptime), updated via set_status_text() -/// * A button (top-right) which rotates the display through 0/90/180/270 -/// * A button (next to it) which clears the circles drawn on the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Status" tab: title + live subsystem state (panel, touch, SD, +/// Ethernet, RTPS, memory/uptime) and the rotate / clear buttons. +/// Touching the screen while this tab is active draws circles (on a +/// transparent overlay). +/// * "Audio" tab: record / play buttons (wired to the example via +/// callbacks) and speaker / microphone volume buttons (acting directly on +/// the BSP), with labels showing the audio state and current volumes. class Gui { public: + /// Callback invoked when the record / play buttons are pressed + using audio_button_callback_t = std::function; /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -67,8 +76,39 @@ class Gui { /// re-aligning the UI to match. Thread-safe. void next_rotation(); + /// Whether the Status tab is currently active (used by the example to + /// only draw circles for touches on that tab). Thread-safe. + /// @return True if the Status tab is the active tab + bool draw_page_active(); + + /// Set the status line on the Audio tab (e.g. "Recording...", "Mic + /// unavailable"). Thread-safe. + /// @param text The text to display + void set_audio_status(std::string_view text); + + /// Set the callback invoked when the record button is pressed + /// @param callback The callback to invoke + void set_record_callback(audio_button_callback_t callback) { + record_callback_ = std::move(callback); + } + + /// Set the callback invoked when the play button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + + /// Show whether a recording is in progress (turns the record button red + /// and changes its symbol to stop). Thread-safe. + /// @param active True while recording + void set_record_active(bool active); + + /// Show whether a playback is in progress (turns the play button green and + /// changes its symbol to stop). Thread-safe. + /// @param active True while playing + void set_play_active(bool active); + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 50; struct Circle { int x{0}; @@ -81,11 +121,16 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); + void init_tabview(); void init_labels(); void init_buttons(); + void init_audio_controls(); void init_circle_layer(); + // update the audio volume label from the BSP's current volumes; called + // with the mutex held + void update_audio_label(); + // the LVGL update task: calls lv_task_handler() under the mutex bool update(std::mutex &m, std::condition_variable &cv); @@ -93,6 +138,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_clicked(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -103,19 +149,36 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *status_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *title_label_{nullptr}; lv_obj_t *status_label_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *record_button_{nullptr}; + lv_obj_t *record_button_label_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *play_button_label_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *mic_down_button_{nullptr}; + lv_obj_t *mic_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t record_callback_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + std::array circles_; size_t next_circle_index_{0}; size_t visible_circle_count_{0}; espp::Task update_task_{{.callback = [this](auto &m, auto &cv) { return update(m, cv); }, - .task_config = {.name = "gui", .stack_size_bytes = 6 * 1024}}}; + // NOTE: rendering the tabview (nested containers + flex layout) uses + // noticeably more stack than a flat UI; 6 KB overflows + .task_config = {.name = "gui", .stack_size_bytes = 12 * 1024}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/esp32-p4-function-ev-board/example/sdkconfig.defaults b/components/esp32-p4-function-ev-board/example/sdkconfig.defaults index 6d3f1dcf8..d18e81de3 100644 --- a/components/esp32-p4-function-ev-board/example/sdkconfig.defaults +++ b/components/esp32-p4-function-ev-board/example/sdkconfig.defaults @@ -51,6 +51,10 @@ CONFIG_LV_THEME_DEFAULT_GROW=y CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=30 # ESP32-P4 Function EV Board BSP configuration -CONFIG_ESP_P4_EV_BOARD_INTERRUPT_STACK_SIZE=4096 -CONFIG_ESP_P4_EV_BOARD_TOUCH_TASK_STACK_SIZE=4096 +# Raised from the 4 KB BSP defaults: the interrupt task and the polled +# touch task run touch-controller I2C and, on a bus error, log through +# espp's fmt color-print path, which needs more than 4 KB of stack. See +# the README. +CONFIG_ESP_P4_EV_BOARD_INTERRUPT_STACK_SIZE=8192 +CONFIG_ESP_P4_EV_BOARD_TOUCH_TASK_STACK_SIZE=8192 CONFIG_ESP_P4_EV_BOARD_AUDIO_TASK_STACK_SIZE=8192 diff --git a/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp b/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp index 207b498c2..fd4a8f148 100644 --- a/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp +++ b/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp @@ -270,13 +270,59 @@ class Esp32P4FunctionEvBoard : public BaseComponent { size_t audio_buffer_size() const; /// Play audio data - /// \param data The audio data to play + /// \param data The audio data to play (16-bit signed mono samples) /// \param num_bytes The number of bytes to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Play audio data - /// \param data The audio data to play - void play_audio(std::span data); + /// \param data The audio data to play (16-bit signed mono samples) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(std::span data); + + ///////////////////////////////////////////////////////////////////////////// + // Microphone + ///////////////////////////////////////////////////////////////////////////// + + /// Alias for the microphone callback, called with recorded audio data + using microphone_callback_t = std::function; + + /// Initialize the microphone (the onboard analog microphone through the + /// ES8311 codec's ADC) and start delivering audio data to the provided + /// callback + /// \param callback The callback to call with recorded audio data (16-bit + /// signed mono samples at audio_sample_rate()) + /// \param task_config The configuration for the microphone task + /// \return true if the microphone was successfully initialized, false + /// otherwise + /// \note The audio subsystem must be initialized first (the ES8311 is a + /// full-duplex codec on a single I2S bus, so the microphone records + /// at the speaker's sample rate) + /// \note The callback runs in the microphone task's context, so the task's + /// stack must be large enough for whatever the callback does with + /// the audio data + bool initialize_microphone(const microphone_callback_t &callback, + const espp::Task::BaseConfig &task_config = {.name = "microphone", + .stack_size_bytes = 4096, + .priority = 10, + .core_id = 1}); + + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100), mapped onto the + /// ES8311 analog microphone gain range (0 dB - +42 dB) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100) + float microphone_volume() const; ///////////////////////////////////////////////////////////////////////////// // uSD Card (4-bit SDMMC) @@ -360,6 +406,7 @@ class Esp32P4FunctionEvBoard : public BaseComponent { bool update_touch(); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); ///////////////////////////////////////////////////////////////////////////// // Display geometry / per-panel parameters @@ -539,6 +586,15 @@ class Esp32P4FunctionEvBoard : public BaseComponent { StreamBufferHandle_t audio_tx_stream{nullptr}; std::atomic has_sound{false}; + // Microphone (ES8311 ADC, full duplex with the speaker) + std::atomic microphone_initialized_{false}; + microphone_callback_t microphone_callback_{nullptr}; + std::unique_ptr microphone_task_{nullptr}; + i2s_chan_handle_t audio_rx_handle{nullptr}; + std::vector audio_rx_buffer; + // microphone volume (percent), mapped onto the ES8311 analog gain range + std::atomic mic_volume_{70.0f}; + // uSD card std::atomic sd_card_initialized_{false}; sdmmc_card_t *sdcard_{nullptr}; diff --git a/components/esp32-p4-function-ev-board/src/audio.cpp b/components/esp32-p4-function-ev-board/src/audio.cpp index 1f9dd137e..2ed293167 100644 --- a/components/esp32-p4-function-ev-board/src/audio.cpp +++ b/components/esp32-p4-function-ev-board/src/audio.cpp @@ -1,6 +1,7 @@ #include "esp32-p4-function-ev-board.hpp" #include +#include #include #include @@ -43,10 +44,12 @@ bool Esp32P4FunctionEvBoard::initialize_audio(uint32_t sample_rate, set_es8311_write(espp::make_i2c_addressed_write(es8311_i2c_device_)); set_es8311_read(espp::make_i2c_addressed_read_register(es8311_i2c_device_)); - // Create the I2S standard channel for playback (TX). MCLK = 256 * fs (default). + // Create the I2S standard channels: TX for playback, RX for the ES8311's + // ADC (initialized on demand by initialize_microphone(); the codec is full + // duplex on this single bus, sharing the clock). MCLK = 256 * fs (default). i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(audio_i2s_port, I2S_ROLE_MASTER); chan_cfg.auto_clear = true; - if (i2s_new_channel(&chan_cfg, &audio_tx_handle, nullptr) != ESP_OK) { + if (i2s_new_channel(&chan_cfg, &audio_tx_handle, &audio_rx_handle) != ESP_OK) { logger_.error("Failed to create I2S channel"); return false; } @@ -124,26 +127,124 @@ void Esp32P4FunctionEvBoard::mute(bool mute) { bool Esp32P4FunctionEvBoard::is_muted() const { return mute_; } -void Esp32P4FunctionEvBoard::play_audio(const uint8_t *data, uint32_t num_bytes) { +size_t Esp32P4FunctionEvBoard::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!audio_initialized_ || !data || num_bytes == 0) { - return; + return 0; } // Non-blocking: enqueue as much as currently fits in the TX stream buffer for // the audio task to drain to I2S. This never blocks the caller, so it is safe - // to call from a touch callback / any task. If the buffer is full the excess - // is dropped (the clip is truncated) rather than stalling the caller. - xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); + // to call from a touch callback / any task. The number of bytes actually + // queued is returned so callers can stream data larger than the buffer. + return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); } -void Esp32P4FunctionEvBoard::play_audio(std::span data) { - play_audio(data.data(), data.size()); +size_t Esp32P4FunctionEvBoard::play_audio(std::span data) { + return play_audio(data.data(), data.size()); } +////////////////////////// +// Microphone Functions // +////////////////////////// + +// Map a 0-100% microphone volume onto the ES8311's analog microphone gain +// steps (ES8311_MIC_GAIN_0DB .. ES8311_MIC_GAIN_42DB, 6 dB apart) +static es8311_mic_gain_t microphone_gain_from_volume(float volume) { + int step = static_cast(std::lround(volume / 100.0f * 7.0f)); + step = std::clamp(step, static_cast(ES8311_MIC_GAIN_0DB), + static_cast(ES8311_MIC_GAIN_42DB)); + return static_cast(step); +} + +bool Esp32P4FunctionEvBoard::initialize_microphone(const microphone_callback_t &callback, + const espp::Task::BaseConfig &task_config) { + logger_.info("Initializing microphone"); + if (microphone_initialized_) { + logger_.warn("Microphone already initialized, not initializing again!"); + return false; + } + if (!audio_initialized_) { + logger_.error("The audio subsystem must be initialized first: the ES8311 is a full-duplex " + "codec on a single I2S bus"); + return false; + } + if (!callback) { + logger_.error("A callback is required to receive the recorded audio data"); + return false; + } + microphone_callback_ = callback; + + // The RX channel shares the TX BCLK/WS in full-duplex mode. Receive in + // stereo (both 16-bit slots of every frame) even though the codec's ADC is + // mono: a mono RX slot configuration in this full-duplex setup does not + // deliver one sample per frame (each sample comes through twice, i.e. at + // half speed); capturing both slots gives a deterministic L,R word layout + // and the microphone task keeps only the left slot, where the ES8311 + // drives its ADC data. + i2s_std_config_t rx_cfg = audio_std_cfg; + rx_cfg.slot_cfg = + I2S_STD_PHILIP_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_STEREO); + if (i2s_channel_init_std_mode(audio_rx_handle, &rx_cfg) != ESP_OK) { + logger_.error("Failed to init I2S RX std mode"); + return false; + } + // one update period's worth of stereo frames (NUM_CHANNELS is 2) + audio_rx_buffer.resize(calc_audio_buffer_size(audio_sample_rate())); + if (i2s_channel_enable(audio_rx_handle) != ESP_OK) { + logger_.error("Failed to enable I2S RX channel"); + return false; + } + + // Enable the codec's ADC path alongside the running DAC and apply the + // stored microphone gain + es8311_codec_ctrl_state(AUDIO_HAL_CODEC_MODE_BOTH, AUDIO_HAL_CTRL_START); + es8311_set_mic_gain(microphone_gain_from_volume(mic_volume_)); + + using namespace std::placeholders; + microphone_task_ = espp::Task::make_unique({ + .callback = std::bind(&Esp32P4FunctionEvBoard::microphone_task_callback, this, _1, _2, _3), + .task_config = task_config, + }); + + microphone_initialized_ = true; + + return microphone_task_->start(); +} + +bool Esp32P4FunctionEvBoard::microphone_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + size_t bytes_read = 0; + auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), + &bytes_read, portMAX_DELAY); + if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { + // compact the L,R word pairs down to mono in place, keeping the left + // slot (the ES8311's ADC data) + auto *samples = reinterpret_cast(audio_rx_buffer.data()); + size_t num_frames = bytes_read / (2 * sizeof(int16_t)); + for (size_t i = 0; i < num_frames; i++) { + samples[i] = samples[2 * i]; + } + microphone_callback_(audio_rx_buffer.data(), num_frames * sizeof(int16_t)); + } + return false; // don't stop the task +} + +void Esp32P4FunctionEvBoard::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (microphone_initialized_) { + es8311_set_mic_gain(microphone_gain_from_volume(mic_volume_)); + } +} + +float Esp32P4FunctionEvBoard::microphone_volume() const { return mic_volume_; } + bool Esp32P4FunctionEvBoard::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); int buffer_size = audio_tx_buffer.size(); available = std::min(available, buffer_size); + // only ever hand whole 16-bit samples to I2S; a partial sample would shift + // the framing of everything after it + available &= ~static_cast(1); uint8_t *tx_buf = audio_tx_buffer.data(); memset(tx_buf, 0, buffer_size); if (available == 0) { diff --git a/components/m5stack-cardputer/README.md b/components/m5stack-cardputer/README.md index 04955f53f..b428718d6 100644 --- a/components/m5stack-cardputer/README.md +++ b/components/m5stack-cardputer/README.md @@ -24,8 +24,9 @@ singleton hardware abstraction for initializing and using the subsystems: an NS4168 amplifier directly; on the ADV it goes through an ES8311 codec (initialized automatically) into an NS4150B amplifier. - **Microphone**: a recording task delivers 16-bit mono samples to a - callback. Original: SPM1423 PDM microphone; ADV: analog MEMS microphone via - the ES8311 codec's ADC. + callback, with adjustable volume via `microphone_volume()`. Original: + SPM1423 PDM microphone (volume applied in software); ADV: analog MEMS + microphone via the ES8311 codec's ADC (volume applied in the codec). - **uSD card**: SPI-mode micro-SD mounted at `/sdcard`. - **RGB LED**: the StampS3's WS2812 via `espp::Neopixel`. - **Battery**: battery voltage measurement (2:1 divider into ADC1) and an diff --git a/components/m5stack-cardputer/example/README.md b/components/m5stack-cardputer/example/README.md index b55ebb915..744aad43f 100644 --- a/components/m5stack-cardputer/example/README.md +++ b/components/m5stack-cardputer/example/README.md @@ -29,6 +29,8 @@ component to initialize and use the components on the M5Stack Cardputer: | Fn + 2 (F2) | Show / hide the IMU overlay (ADV) | | Fn + 3 (F3) | Start / stop recording from the microphone (ADV) | | Fn + 4 (F4) | Play back / stop the recording (ADV) | +| Fn + 5 / 6 (F5 / F6) | Speaker volume down / up | +| Fn + 7 / 8 (F7 / F8) | Microphone volume down / up (75% = 0 dB) | | Fn + number row | Show F1-F12 in the status bar | | G0 (BOOT) button | Cycle the RGB LED color | diff --git a/components/m5stack-cardputer/example/main/gui.hpp b/components/m5stack-cardputer/example/main/gui.hpp index d51d7d2c4..403334051 100644 --- a/components/m5stack-cardputer/example/main/gui.hpp +++ b/components/m5stack-cardputer/example/main/gui.hpp @@ -41,6 +41,8 @@ class Gui { "fn+2 (F2) show/hide IMU\n" "fn+3 (F3) record / stop\n" "fn+4 (F4) play recording\n" + "fn+5/6 speaker vol -/+\n" + "fn+7/8 mic vol -/+\n" "G0 button cycle LED color\n" "\n" "While help is open:\n" diff --git a/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp b/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp index 24c619943..80c309076 100644 --- a/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp +++ b/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp @@ -154,6 +154,21 @@ extern "C" void app_main(void) { gui.set_status_text("Playing... (fn+4 stops)"); } play_beep(cardputer, 660.0f); + } else if (event.special == espp::M5StackCardputer::SpecialKey::F5 || + event.special == espp::M5StackCardputer::SpecialKey::F6) { + // speaker volume down / up (the beep gives immediate feedback) + float delta = event.special == espp::M5StackCardputer::SpecialKey::F5 ? -10.0f : 10.0f; + cardputer.volume(cardputer.volume() + delta); + gui.set_status_text(fmt::format("Speaker volume: {:.0f}%", cardputer.volume())); + play_beep(cardputer, 660.0f); + } else if (event.special == espp::M5StackCardputer::SpecialKey::F7 || + event.special == espp::M5StackCardputer::SpecialKey::F8) { + // microphone volume down / up (heard on the next recording) + float delta = event.special == espp::M5StackCardputer::SpecialKey::F7 ? -5.0f : 5.0f; + cardputer.microphone_volume(cardputer.microphone_volume() + delta); + gui.set_status_text( + fmt::format("Mic volume: {:.0f}% (75% = 0 dB)", cardputer.microphone_volume())); + play_beep(cardputer, 660.0f); } else if (event.special != espp::M5StackCardputer::SpecialKey::NONE) { gui.handle_special_key(event.special); gui.set_status_text(espp::M5StackCardputer::special_key_name(event.special)); diff --git a/components/m5stack-cardputer/include/m5stack-cardputer.hpp b/components/m5stack-cardputer/include/m5stack-cardputer.hpp index 45782b13c..5478fe434 100644 --- a/components/m5stack-cardputer/include/m5stack-cardputer.hpp +++ b/components/m5stack-cardputer/include/m5stack-cardputer.hpp @@ -463,6 +463,20 @@ class M5StackCardputer : public BaseComponent { /// \return The microphone sample rate, in Hz uint32_t microphone_sample_rate() const; + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100); 75 is unity (0 dB, + /// the default) + /// \note On the ADV this adjusts the ES8311 codec's digital ADC volume + /// (values above 75 amplify, up to +32 dB at 100); on the original + /// the samples are scaled in software in the microphone task (values + /// above 75 amplify with saturation) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100); 75 is unity + /// (0 dB) + float microphone_volume() const; + ///////////////////////////////////////////////////////////////////////////// // uSD Card ///////////////////////////////////////////////////////////////////////////// @@ -853,6 +867,8 @@ class M5StackCardputer : public BaseComponent { // microphone sample rate (Hz), set by initialize_microphone() std::atomic mic_sample_rate_{0}; + // microphone volume (percent, 75 = unity / 0 dB) + std::atomic mic_volume_{75.0f}; // display std::shared_ptr> display_; diff --git a/components/m5stack-cardputer/src/m5stack-cardputer.cpp b/components/m5stack-cardputer/src/m5stack-cardputer.cpp index 076573db4..0e3ed21b1 100644 --- a/components/m5stack-cardputer/src/m5stack-cardputer.cpp +++ b/components/m5stack-cardputer/src/m5stack-cardputer.cpp @@ -1,4 +1,5 @@ #include +#include #include #include "m5stack-cardputer.hpp" @@ -135,13 +136,14 @@ bool M5StackCardputer::initialize_es8311_microphone() { // (hardware-proven with the esp-box microphone): M5Unified's minimal // sequence uses minimum PGA gain (0x14 = 0x10) and no digital mic gain, // which records at a near-mute level. + const uint8_t adc_volume = static_cast(std::lround(mic_volume_ / 100.0f * 255.0f)); const uint8_t init[][2] = { - {0x0E, 0x02}, // SYSTEM: enable analog PGA / ADC modulator - {0x14, 0x1A}, // SYSTEM: select Mic1p-Mic1n, raised analog PGA gain - {0x15, 0x40}, // ADC: soft-ramp / ALC configuration - {0x16, 0x24}, // ADC: mic digital gain scale - {0x17, 0xBF}, // ADC: volume 0 dB - {0x1C, 0x6A}, // ADC: equalizer bypass + {0x0E, 0x02}, // SYSTEM: enable analog PGA / ADC modulator + {0x14, 0x1A}, // SYSTEM: select Mic1p-Mic1n, raised analog PGA gain + {0x15, 0x40}, // ADC: soft-ramp / ALC configuration + {0x16, 0x24}, // ADC: mic digital gain scale + {0x17, adc_volume}, // ADC: volume (0x00 = -95.5 dB, 0xBF = 0 dB, 0xFF = +32 dB) + {0x1C, 0x6A}, // ADC: equalizer bypass }; return std::all_of(std::begin(init), std::end(init), [this](const auto &entry) { if (!es8311_write(entry[0], entry[1])) { diff --git a/components/m5stack-cardputer/src/microphone.cpp b/components/m5stack-cardputer/src/microphone.cpp index d000927c6..e93c0384c 100644 --- a/components/m5stack-cardputer/src/microphone.cpp +++ b/components/m5stack-cardputer/src/microphone.cpp @@ -1,3 +1,6 @@ +#include +#include + #include "m5stack-cardputer.hpp" using namespace espp; @@ -106,6 +109,17 @@ bool M5StackCardputer::initialize_microphone(const microphone_callback_t &callba uint32_t M5StackCardputer::microphone_sample_rate() const { return mic_sample_rate_; } +void M5StackCardputer::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (variant() == Variant::ADV && microphone_initialized_) { + // the ES8311 scales in hardware; initialize_es8311_microphone() applies + // the stored volume when the microphone is initialized later + es8311_write(0x17, static_cast(std::lround(mic_volume_ / 100.0f * 255.0f))); + } +} + +float M5StackCardputer::microphone_volume() const { return mic_volume_; } + bool M5StackCardputer::microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { size_t bytes_read = 0; @@ -121,6 +135,19 @@ bool M5StackCardputer::microphone_task_callback(std::mutex &m, std::condition_va samples[i] = samples[2 * i]; } bytes_read = num_frames * sizeof(int16_t); + } else { + // the original's PDM microphone has no hardware gain, so apply the + // microphone volume in software (75% = unity; the ADV scales in the + // codec instead) + float scale = mic_volume_ / 75.0f; + if (scale != 1.0f) { + auto *samples = reinterpret_cast(audio_rx_buffer.data()); + size_t num_samples = bytes_read / sizeof(int16_t); + for (size_t i = 0; i < num_samples; i++) { + auto v = static_cast(samples[i] * scale); + samples[i] = static_cast(std::clamp(v, INT16_MIN, INT16_MAX)); + } + } } microphone_callback_(audio_rx_buffer.data(), bytes_read); } diff --git a/components/m5stack-tab5/example/README.md b/components/m5stack-tab5/example/README.md index 69419de87..49f7b9a0a 100644 --- a/components/m5stack-tab5/example/README.md +++ b/components/m5stack-tab5/example/README.md @@ -9,7 +9,14 @@ This example demonstrates the comprehensive functionality of the M5Stack Tab5 de ### Core Systems - **5" 720p MIPI-DSI Display**: Initialization and brightness control - **GT911 Multi-Touch Controller**: Touch event handling with callbacks -- **Dual Audio System**: ES8388 codec + ES7210 AEC with recording/playback +- **Dual Audio System**: ES8388 codec + ES7210 AEC with recording/playback. + The on-screen audio row (bottom-left) records from the microphones, plays + the recording back through the speaker, and adjusts the speaker / + microphone volumes; the recording buffer prefers PSRAM and the measured + capture rate is logged when a recording stops. Because the speaker is + mono, the recorded stereo (mic 1 / mic 2) is downmixed to both channels + before playback, and the per-channel peak amplitude is logged so you can + tell a silent capture (peak near 0) from a playback issue. - **BMI270 6-axis IMU**: Real-time motion sensing - **Battery Management**: INA226 power monitoring with charging control @@ -17,6 +24,20 @@ This example demonstrates the comprehensive functionality of the M5Stack Tab5 de - **microSD Card**: File system support with SDIO interface - **Real-Time Clock**: RX8130CE with alarm and wake-up features +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_M5STACK_TAB5_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. + ## Hardware Requirements - M5Stack Tab5 development board diff --git a/components/m5stack-tab5/example/main/gui.cpp b/components/m5stack-tab5/example/main/gui.cpp index ea312e8e0..0c883583f 100644 --- a/components/m5stack-tab5/example/main/gui.cpp +++ b/components/m5stack-tab5/example/main/gui.cpp @@ -1,18 +1,15 @@ +#include #include #include "gui.hpp" void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); - init_label(); - init_gravity_lines(); - init_buttons(); + init_tabview(); + init_draw_tab(); + init_status_tab(); + init_audio_tab(); init_circle_layer(); - // disable scrolling on the screen (so that it doesn't behave weirdly when - // rotated and drawing with your finger) - lv_obj_set_scrollbar_mode(lv_screen_active(), LV_SCROLLBAR_MODE_OFF); - lv_obj_clear_flag(lv_screen_active(), LV_OBJ_FLAG_SCROLLABLE); } void Gui::deinit_ui() { @@ -20,31 +17,72 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &tab5 = espp::M5StackTab5::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, tab5.display_width(), tab5.display_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(0, 0, 0), 0); +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); + status_tab_ = lv_tabview_add_tab(tabview_, "Status"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } -void Gui::init_label() { - label_ = lv_label_create(lv_screen_active()); +void Gui::init_draw_tab() { + // instructions on the left, with the rotate / brightness / clear buttons + // on the right + label_ = lv_label_create(draw_tab_); + lv_label_set_long_mode(label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(label_, lv_pct(70)); lv_label_set_text(label_, ""); lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 0, 0); lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_LEFT, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + int y_offset; + }; + const ButtonSpec buttons[] = { + {&rotate_button_, LV_SYMBOL_REFRESH, 0}, + {&brightness_button_, LV_SYMBOL_EYE_OPEN, 60}, + {&clear_button_, LV_SYMBOL_TRASH, 120}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(draw_tab_); + lv_obj_set_size(*spec.button, 50, 50); + lv_obj_align(*spec.button, LV_ALIGN_TOP_RIGHT, 0, spec.y_offset); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } } -void Gui::init_gravity_lines() { - auto &tab5 = espp::M5StackTab5::get(); +void Gui::init_status_tab() { + status_label_ = lv_label_create(status_tab_); + lv_label_set_long_mode(status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(status_label_, lv_pct(100)); + lv_label_set_text(status_label_, "Waiting for data..."); + lv_obj_align(status_label_, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_set_style_text_align(status_label_, LV_TEXT_ALIGN_LEFT, 0); lv_style_init(&kalman_line_style_); lv_style_set_line_width(&kalman_line_style_, 8); lv_style_set_line_color(&kalman_line_style_, lv_palette_main(LV_PALETTE_BLUE)); lv_style_set_line_rounded(&kalman_line_style_, true); - kalman_line_ = lv_line_create(lv_screen_active()); + kalman_line_ = lv_line_create(status_tab_); kalman_line_points_[0] = {0, 0}; - kalman_line_points_[1] = {tab5.display_width(), tab5.display_height()}; + kalman_line_points_[1] = {0, 0}; lv_line_set_points(kalman_line_, kalman_line_points_, 2); lv_obj_add_style(kalman_line_, &kalman_line_style_, 0); @@ -53,56 +91,82 @@ void Gui::init_gravity_lines() { lv_style_set_line_color(&madgwick_line_style_, lv_palette_main(LV_PALETTE_RED)); lv_style_set_line_rounded(&madgwick_line_style_, true); - madgwick_line_ = lv_line_create(lv_screen_active()); + madgwick_line_ = lv_line_create(status_tab_); madgwick_line_points_[0] = {0, 0}; - madgwick_line_points_[1] = {tab5.display_width(), tab5.display_height()}; + madgwick_line_points_[1] = {0, 0}; lv_line_set_points(madgwick_line_, madgwick_line_points_, 2); lv_obj_add_style(madgwick_line_, &madgwick_line_style_, 0); } -void Gui::init_buttons() { - // a button in the top left which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(rotate_button_, 50, 50); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_LEFT, 0, 0); - lv_obj_t *rotate_label = lv_label_create(rotate_button_); - lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); - lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); - - // a button next to it which cycles the backlight brightness through - // 25/50/75/100% - brightness_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(brightness_button_, 50, 50); - lv_obj_align(brightness_button_, LV_ALIGN_TOP_LEFT, 60, 0); - lv_obj_t *brightness_label = lv_label_create(brightness_button_); - lv_label_set_text(brightness_label, LV_SYMBOL_EYE_OPEN); - lv_obj_align(brightness_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(brightness_button_, event_callback, LV_EVENT_PRESSED, this); - - // a button in the top right which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(clear_button_, 50, 50); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 0); - lv_obj_t *clear_label = lv_label_create(clear_button_); - lv_label_set_text(clear_label, LV_SYMBOL_TRASH); - lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); +void Gui::init_audio_tab() { + // a column: status line, volume line, then the buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 12, 0); + + audio_status_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_status_label_, lv_pct(100)); + lv_label_set_text(audio_status_label_, "Idle"); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 12, 0); + lv_obj_set_style_pad_row(row, 12, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&record_button_, LV_SYMBOL_AUDIO}, {&play_button_, LV_SYMBOL_PLAY}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + {&mic_down_button_, LV_SYMBOL_MINUS}, {&mic_up_button_, LV_SYMBOL_PLUS}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 60, 60); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } + // remember the record / play button labels so set_record_active / + // set_play_active can swap their symbols + record_button_label_ = lv_obj_get_child(record_button_, 0); + play_button_label_ = lv_obj_get_child(play_button_, 0); + // color the volume buttons so the speaker and microphone pairs are + // distinguishable from each other + lv_obj_set_style_bg_color(mic_down_button_, lv_palette_main(LV_PALETTE_TEAL), 0); + lv_obj_set_style_bg_color(mic_up_button_, lv_palette_main(LV_PALETTE_TEAL), 0); } -void Gui::init_circle_layer() { +void Gui::update_audio_label() { auto &tab5 = espp::M5StackTab5::get(); + lv_label_set_text_fmt(audio_label_, + "Speaker %d%% (" LV_SYMBOL_VOLUME_MID "/" LV_SYMBOL_VOLUME_MAX + ")\nMic %d%% (teal " LV_SYMBOL_MINUS "/" LV_SYMBOL_PLUS ")", + static_cast(tab5.volume()), + static_cast(tab5.microphone_volume())); +} + +void Gui::init_circle_layer() { + // a transparent, click-through overlay above the tabview which shows the + // touch trail (only populated while the Draw tab is active) circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, tab5.display_width(), tab5.display_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); lv_obj_set_style_bg_opa(circle_layer_, LV_OPA_TRANSP, 0); - lv_obj_set_style_border_width(circle_layer_, 0, 0); - lv_obj_set_style_outline_width(circle_layer_, 0, 0); - lv_obj_set_style_shadow_width(circle_layer_, 0, 0); lv_obj_add_event_cb(circle_layer_, draw_circle_layer, LV_EVENT_DRAW_MAIN, this); lv_obj_move_foreground(circle_layer_); } @@ -111,6 +175,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -126,13 +194,59 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_PRESSED: gui->on_pressed(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_pressed(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &tab5 = espp::M5StackTab5::get(); + if (target == record_button_) { + logger_.info("Record button pressed"); + if (record_callback_) { + record_callback_(); + } + return; + } + if (target == play_button_) { + logger_.info("Play button pressed"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + tab5.volume(tab5.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", tab5.volume()); + update_audio_label(); + return; + } + if (target == mic_down_button_ || target == mic_up_button_) { + float delta = target == mic_down_button_ ? -10.0f : 10.0f; + tab5.microphone_volume(tab5.microphone_volume() + delta); + logger_.info("Microphone volume: {:.0f}%", tab5.microphone_volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button pressed"); next_rotation(); @@ -145,21 +259,53 @@ void Gui::on_pressed(lv_event_t *e) { } } +void Gui::set_record_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(record_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_AUDIO); + lv_obj_set_style_bg_color( + record_button_, active ? lv_palette_main(LV_PALETTE_RED) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_play_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(play_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_PLAY); + lv_obj_set_style_bg_color( + play_button_, active ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_audio_status(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(audio_status_label_, std::string(text).c_str()); +} + void Gui::set_label_text(std::string_view text) { std::lock_guard lock(mutex_); lv_label_set_text(label_, std::string(text).c_str()); } +void Gui::set_status_text(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(status_label_, std::string(text).c_str()); +} + +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::next_rotation() { std::lock_guard lock(mutex_); - auto &tab5 = espp::M5StackTab5::get(); clear_circles_impl(); auto rotation = lv_display_get_rotation(lv_display_get_default()); rotation = static_cast((static_cast(rotation) + 1) % 4); lv_display_set_rotation(lv_display_get_default(), rotation); // update the size of the screen-filling objects - lv_obj_set_size(background_, tab5.rotated_display_width(), tab5.rotated_display_height()); - lv_obj_set_size(circle_layer_, tab5.rotated_display_width(), tab5.rotated_display_height()); + int width = lv_display_get_horizontal_resolution(lv_display_get_default()); + int height = lv_display_get_vertical_resolution(lv_display_get_default()); + lv_obj_set_size(tabview_, width, height); + lv_obj_set_size(circle_layer_, width, height); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_invalidate(circle_layer_); } @@ -183,7 +329,6 @@ void Gui::set_madgwick_down(float vx, float vy) { } void Gui::set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, float vy) { - auto &tab5 = espp::M5StackTab5::get(); // remap the vector according to the current display rotation so that the // line always points toward physical "down" auto rotation = lv_display_get_rotation(lv_display_get_default()); @@ -197,9 +342,16 @@ void Gui::set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, fl std::swap(vx, vy); vy = -vy; } - // draw the line from the center of the screen toward "down" - int x0 = tab5.rotated_display_width() / 2; - int y0 = tab5.rotated_display_height() / 2; + // draw the line from the center of the Status tab's content area toward + // "down" (fall back to the display size if the layout hasn't run yet) + int w = lv_obj_get_content_width(status_tab_); + int h = lv_obj_get_content_height(status_tab_); + if (w <= 0 || h <= 0) { + w = lv_display_get_horizontal_resolution(lv_display_get_default()); + h = lv_display_get_vertical_resolution(lv_display_get_default()) - TAB_BAR_HEIGHT; + } + int x0 = w / 2; + int y0 = h / 2; points[0] = {static_cast(x0), static_cast(y0)}; points[1] = {static_cast(x0 + 50 * vx), static_cast(y0 + 50 * vy)}; diff --git a/components/m5stack-tab5/example/main/gui.hpp b/components/m5stack-tab5/example/main/gui.hpp index a6f3c85aa..c5d182c37 100644 --- a/components/m5stack-tab5/example/main/gui.hpp +++ b/components/m5stack-tab5/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -22,16 +23,22 @@ /// dispatched through a single static trampoline (event_callback) into /// member functions, keeping all UI logic inside the class. /// -/// For this example the Gui shows: -/// * A label with instructions and live battery / RTC / IMU data -/// * Two lines showing the direction of gravity ("down") as computed by a -/// Kalman filter (blue) and a Madgwick filter (red) -/// * A button (top-left) which rotates the display through 0/90/180/270 -/// * A button (next to it) which cycles the backlight through 25/50/75/100% -/// * A button (top-right) which clears the circles drawn on the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Draw" tab: instructions, plus the rotate, brightness, and clear +/// buttons. Touching the screen while this tab is active draws circles +/// (on a transparent overlay) and plays a click. +/// * "Status" tab: live battery / RTC / IMU text and two lines showing the +/// direction of gravity ("down") as computed by a Kalman filter (blue) +/// and a Madgwick filter (red). +/// * "Audio" tab: record / play buttons (wired to the example via +/// callbacks) and speaker / microphone volume buttons (acting directly on +/// the BSP), with labels showing the audio state and current volumes. class Gui { public: + /// Callback invoked when the record / play buttons are pressed + using audio_button_callback_t = std::function; + /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -50,10 +57,20 @@ class Gui { deinit_ui(); } - /// Set the text of the main label. Thread-safe. + /// Set the instruction text shown on the Draw tab. Thread-safe. /// @param text The text to display void set_label_text(std::string_view text); + /// Set the live battery / RTC / IMU text shown on the Status tab. + /// Thread-safe. + /// @param text The text to display + void set_status_text(std::string_view text); + + /// Whether the Draw tab is currently active (used by the example to only + /// draw circles / play clicks for touches on that tab). Thread-safe. + /// @return True if the Draw tab is the active tab + bool draw_page_active(); + /// Draw a circle at the given screen coordinates, replacing the oldest /// circle if the maximum number are already visible. Thread-safe. /// @param x The x coordinate (screen space) @@ -87,8 +104,34 @@ class Gui { /// @param vy The y component of the gravity vector void set_madgwick_down(float vx, float vy); + /// Set the callback invoked when the record button is pressed + /// @param callback The callback to invoke + void set_record_callback(audio_button_callback_t callback) { + record_callback_ = std::move(callback); + } + + /// Set the callback invoked when the play button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + + /// Show whether a recording is in progress (turns the record button red + /// and changes its symbol to stop). Thread-safe. + /// @param active True while recording + void set_record_active(bool active); + + /// Show whether a playback is in progress (turns the play button green and + /// changes its symbol to stop). Thread-safe. + /// @param active True while playing + void set_play_active(bool active); + + /// Set the status line on the Audio tab (e.g. "Recording...", "Mic + /// unavailable"). Thread-safe. + /// @param text The text to display + void set_audio_status(std::string_view text); + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 50; struct Circle { int x{0}; @@ -101,14 +144,19 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); - void init_label(); - void init_gravity_lines(); - void init_buttons(); + void init_tabview(); + void init_draw_tab(); + void init_status_tab(); + void init_audio_tab(); void init_circle_layer(); + // update the audio volume label from the BSP's current volumes; called + // with the mutex held + void update_audio_label(); + // update the given "down" line from a vector in the unrotated display - // frame, remapping for the current display rotation + // frame, remapping for the current display rotation; called with the + // mutex held void set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, float vy); // the LVGL update task: calls lv_task_handler() under the mutex @@ -118,6 +166,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_pressed(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -128,15 +177,32 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *draw_tab_{nullptr}; + lv_obj_t *status_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *label_{nullptr}; + lv_obj_t *status_label_{nullptr}; lv_obj_t *kalman_line_{nullptr}; lv_obj_t *madgwick_line_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *brightness_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *record_button_{nullptr}; + lv_obj_t *record_button_label_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *play_button_label_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *mic_down_button_{nullptr}; + lv_obj_t *mic_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t record_callback_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + lv_style_t kalman_line_style_; lv_style_t madgwick_line_style_; lv_point_precise_t kalman_line_points_[2]; @@ -154,7 +220,7 @@ class Gui { espp::Task update_task_{ {.callback = [this](auto &m, auto &cv) { return update(m, cv); }, .task_config = { - .name = "gui", .stack_size_bytes = 10 * 1024, .priority = 20, .core_id = 1}}}; + .name = "gui", .stack_size_bytes = 12 * 1024, .priority = 20, .core_id = 1}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp index 532a5e246..a19f5fa86 100644 --- a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp +++ b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp @@ -7,11 +7,18 @@ * and communication interfaces. */ +#include +#include #include #include +#include +#include #include #include +#include +#include + #include "m5stack-tab5.hpp" #include "gui.hpp" @@ -25,6 +32,21 @@ static std::vector audio_bytes; static bool load_audio(size_t &out_size, size_t &out_sample_rate); static void play_click(espp::M5StackTab5 &tab5); +// Audio recording state (written by the recording callback, read/controlled +// from the GUI button callbacks and main loop). The recorded data is 16-bit +// interleaved stereo at the current audio sample rate. +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 2; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static int64_t recording_start_us = 0; +static int64_t recording_last_us = 0; + extern "C" void app_main(void) { espp::Logger logger({.tag = "M5Stack Tab5 Example", .level = espp::Logger::Verbosity::INFO}); logger.info("Starting example!"); @@ -181,6 +203,11 @@ extern "C" void app_main(void) { return; } + // unmute the audio and set the volume to 60% (do this before the GUI is + // created so its volume label shows the right value) + tab5.mute(false); + tab5.volume(60.0f); + // create the GUI: builds the UI (label, buttons, gravity lines, circle // layer) and starts the task which updates LVGL. All of its public methods // are thread-safe, so the touch callback, button callback, and data display @@ -188,8 +215,9 @@ extern "C" void app_main(void) { logger.info("Setting up LVGL UI..."); static Gui gui({.log_level = espp::Logger::Verbosity::INFO}); static const std::string instructions = - fmt::format("\n\n\n\nTouch the screen!\nPress the {} button to clear circles.\nPress the {} " - "button to rotate the display.\nPress the {} button to cycle the brightness.", + fmt::format("Touch the screen to draw!\nPress the {} button to clear circles.\nPress the " + "{} button to rotate the display.\nPress the {} button to cycle the " + "brightness.\nThe Status and Audio tabs show the other subsystems.", LV_SYMBOL_TRASH, LV_SYMBOL_REFRESH, LV_SYMBOL_EYE_OPEN); gui.set_label_text(instructions); @@ -222,14 +250,19 @@ extern "C" void app_main(void) { if (touchpad_data.btn_state) { gui.clear_circles(); } - // if there is a touch point, draw a circle and play a click sound - if (touchpad_data.num_touch_points > 0) { + // if there is a touch point on the Draw tab, draw a circle and play a + // click sound (touches on the other tabs go to their widgets) + if (touchpad_data.num_touch_points > 0 && gui.draw_page_active()) { play_click(tab5); gui.draw_circle(touchpad_data.x, touchpad_data.y, 10); } } }; logger.info("Initializing touch..."); + // NOTE: this example raises the BSP interrupt-task stack size via + // sdkconfig.defaults (CONFIG_M5STACK_TAB5_INTERRUPT_STACK_SIZE=8192); the + // touch controller is read from that task and its error-logging path + // needs more than the 4 KB BSP default. See the example README. if (!tab5.initialize_touch(touch_callback)) { logger.error("Failed to initialize touch!"); return; @@ -247,13 +280,94 @@ extern "C" void app_main(void) { logger.info("Setting audio sample rate to {} Hz", wav_sample_rate); tab5.audio_sample_rate(wav_sample_rate); - // unmute the audio and set the volume to 60% - tab5.mute(false); - tab5.volume(60.0f); - // set the brightness to 75% tab5.brightness(75.0f); + // Keep the analog microphone gain modest: the ES7210 front-end develops a + // high-frequency whine as the analog gain is raised, so the loudness comes + // from the RMS software makeup gain applied to the recording on stop (see + // below) rather than from the analog stage. The mic +/- buttons still adjust + // the analog gain if desired. + tab5.microphone_volume(40.0f); + + // Allocate the recording buffer (16-bit interleaved stereo at the current + // sample rate): prefer PSRAM, fall back to a couple of seconds in internal + // RAM + size_t bytes_per_second = tab5.audio_sample_rate() * 2 * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + gui.set_audio_status("No recording buffer"); + logger.warn("Could not allocate a recording buffer; recording disabled"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz stereo)", recording_capacity / 1024, + recording_capacity / bytes_per_second, tab5.audio_sample_rate()); + } + + // The recording callback appends the recorded stereo frames to the buffer + // and auto-stops when it is full (the main loop notices and updates the + // GUI) + auto record_data_callback = [](const uint8_t *data, size_t length) { + if (!recording) { + return; + } + size_t offset = recording_len; + size_t to_copy = std::min(length, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = esp_timer_get_time(); + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + recording = false; + } + }; + + // The record button toggles recording; the play button toggles playback of + // the recording (streamed to the speaker by the main loop) + gui.set_record_callback([&]() { + if (recording_capacity == 0) { + logger.warn("Recording unavailable (no buffer)"); + gui.set_audio_status("Recording unavailable (see log)"); + return; + } + if (recording) { + recording = false; // the main loop stops the BSP recording and logs + } else { + playing = false; + gui.set_play_active(false); + recording_len = 0; + recording_start_us = esp_timer_get_time(); + recording_last_us = recording_start_us; + recording = true; + tab5.start_audio_recording(record_data_callback); + gui.set_record_active(true); + gui.set_audio_status("Recording..."); + } + }); + gui.set_play_callback([&]() { + if (playing) { + playing = false; + gui.set_play_active(false); + gui.set_audio_status("Playback stopped"); + } else if (recording_len > 0) { + recording = false; + playing = true; + gui.set_play_active(true); + gui.set_audio_status("Playing..."); + } else { + logger.info("Nothing recorded yet; press the record button first"); + gui.set_audio_status("Nothing recorded yet"); + } + }); + // make a task to read out various data such as IMU, battery monitoring, etc. // and print it to screen logger.info("Starting data display task..."); @@ -262,7 +376,7 @@ extern "C" void app_main(void) { // sleep first in case we don't get IMU data and need to exit early { std::unique_lock lock(m); - cv.wait_for(lock, 20ms); + cv.wait_for(lock, 10ms); } static auto &tab5 = espp::M5StackTab5::get(); static auto imu = tab5.imu(); @@ -331,14 +445,14 @@ extern "C" void app_main(void) { vx = -vx; vy = -vy; - std::string text = fmt::format("{}\n\n\n\n\n", instructions); + std::string text; text += battery_text; text += rtc_text; text += imu_text; // update the GUI with the new data; the Gui handles remapping the // vectors for the current display rotation - gui.set_label_text(text); + gui.set_status_text(text); gui.set_kalman_down(gravity_vector.x, gravity_vector.y); gui.set_madgwick_down(vx, vy); @@ -352,9 +466,83 @@ extern "C" void app_main(void) { }}); imu_task.start(); - // loop forever + // Main loop: stream any active playback to the speaker and notice when a + // recording stops (either button press or the buffer filling up) + size_t play_offset = 0; + bool was_recording = false; while (true) { - std::this_thread::sleep_for(1s); + // feed the active playback in chunks, advancing by however much the + // speaker's stream buffer accepted + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_play_active(false); + gui.set_audio_status("Playback done"); + logger.info("Playback done"); + } else { + play_offset += tab5.play_audio(recording_buffer + play_offset, + std::min(len - play_offset, 16384)); + } + } else { + play_offset = 0; + } + // notice when the recording stopped (button press or buffer full) + bool now_recording = recording; + if (was_recording && !now_recording) { + tab5.stop_audio_recording(); + gui.set_record_active(false); + gui.set_audio_status(fmt::format("Recorded {:.1f}s ({} plays)", + static_cast(recording_len) / + (tab5.audio_sample_rate() * 2 * sizeof(int16_t)), + LV_SYMBOL_PLAY)); + // report the measured capture rate: stereo frames recorded over the + // wall clock they took to arrive should match the nominal sample rate + size_t num_frames = recording_len / (2 * sizeof(int16_t)); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_frames / elapsed_s : 0.0f; + // Post-process the recording for playback on the Tab5's MONO speaker. + // The ES7210 records 16-bit interleaved stereo (mic 1 -> left slot, mic 2 + // -> right); downmix each frame to the average (the speaker plays one I2S + // slot, so the result is written to both). The captured level is low, so + // rather than driving the analog gain hot (which whines), remove the DC + // offset and apply an RMS-normalized software makeup gain: this makes the + // recording play back at a consistent, audible level comparable to the + // click WAV, without a high analog mic gain or a high speaker volume. + auto *samples = reinterpret_cast(recording_buffer); + int16_t peak = 0; + int64_t sum = 0; + for (size_t i = 0; i < num_frames; i++) { + int32_t mono = (static_cast(samples[2 * i]) + samples[2 * i + 1]) / 2; + samples[2 * i] = static_cast(mono); + peak = std::max(peak, static_cast(std::abs(mono))); + sum += mono; + } + int32_t dc = num_frames ? static_cast(sum / static_cast(num_frames)) : 0; + int64_t sum_sq = 0; + for (size_t i = 0; i < num_frames; i++) { + int32_t v = samples[2 * i] - dc; + sum_sq += static_cast(v) * v; + } + double rms = num_frames ? std::sqrt(static_cast(sum_sq) / num_frames) : 0.0; + // target ~-15 dBFS leaves headroom; cap the gain so a near-silent capture + // is not blown up into noise + static constexpr double target_rms = 5500.0; + double gain = rms > 1.0 ? std::clamp(target_rms / rms, 1.0, 64.0) : 1.0; + for (size_t i = 0; i < num_frames; i++) { + int32_t v = static_cast((samples[2 * i] - dc) * gain); + int16_t mono = static_cast(std::clamp(v, -32768, 32767)); + samples[2 * i] = mono; + samples[2 * i + 1] = mono; + } + logger.info("Recorded {} frames in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal); " + "peak={} dc={} rms={:.0f}; applied makeup gain {:.1f}x", + num_frames, elapsed_s, effective_hz, tab5.audio_sample_rate(), peak, dc, rms, + gain); + } + was_recording = now_recording; + std::this_thread::sleep_for(50ms); } //! [m5stack tab5 example] } diff --git a/components/m5stack-tab5/example/sdkconfig.defaults b/components/m5stack-tab5/example/sdkconfig.defaults index a48e51abc..afef7c936 100644 --- a/components/m5stack-tab5/example/sdkconfig.defaults +++ b/components/m5stack-tab5/example/sdkconfig.defaults @@ -36,7 +36,10 @@ CONFIG_FREERTOS_USE_TRACE_FACILITY=y CONFIG_FREERTOS_USE_STATS_FORMATTING_FUNCTIONS=y # M5Stack Tab5 BSP configuration -CONFIG_M5STACK_TAB5_INTERRUPT_STACK_SIZE=4096 +# Raised from the 4 KB BSP default: the interrupt task runs the touch- +# controller I2C reads and, on a bus error, logs through espp's fmt +# color-print path, which needs more than 4 KB of stack. See the README. +CONFIG_M5STACK_TAB5_INTERRUPT_STACK_SIZE=8192 CONFIG_M5STACK_TAB5_AUDIO_TASK_STACK_SIZE=8192 # PSRAM configuration diff --git a/components/m5stack-tab5/include/m5stack-tab5.hpp b/components/m5stack-tab5/include/m5stack-tab5.hpp index 00710050b..f42d07fe1 100755 --- a/components/m5stack-tab5/include/m5stack-tab5.hpp +++ b/components/m5stack-tab5/include/m5stack-tab5.hpp @@ -310,22 +310,44 @@ class M5StackTab5 : public BaseComponent { size_t audio_buffer_size() const; /// Play audio data - /// \param data The audio data to play + /// \param data The audio data to play (16-bit signed interleaved stereo) /// \param num_bytes The number of bytes to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Play audio data - /// \param data The audio data to play - void play_audio(std::span data); + /// \param data The audio data to play (16-bit signed interleaved stereo) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(std::span data); /// Start recording audio - /// \param callback Function to call with recorded audio data + /// \param callback Function to call with recorded audio data: 16-bit + /// signed interleaved stereo (ES7210 microphone 1 on the left slot, + /// microphone 2 on the right) at audio_sample_rate() /// \return True if recording started successfully + /// \note The callback runs in the audio task's context bool start_audio_recording(std::function callback); /// Stop recording audio void stop_audio_recording(); + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100), mapped onto the + /// ES7210 analog microphone gain range (0 dB - +37.5 dB) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100) + float microphone_volume() const; + ///////////////////////////////////////////////////////////////////////////// // IMU & Sensors ///////////////////////////////////////////////////////////////////////////// @@ -504,6 +526,7 @@ class M5StackTab5 : public BaseComponent { bool update_touch(); bool update_battery_status(); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); // Hardware pin definitions based on Tab5 specifications @@ -728,10 +751,13 @@ class M5StackTab5 : public BaseComponent { // Audio system std::atomic audio_initialized_{false}; std::atomic volume_{50.0f}; + // microphone volume (percent), mapped onto the ES7210 analog gain range + std::atomic mic_volume_{70.0f}; std::atomic mute_{false}; std::shared_ptr> es8388_i2c_device_; std::shared_ptr> es7210_i2c_device_; std::unique_ptr audio_task_{nullptr}; + std::unique_ptr microphone_task_{nullptr}; i2s_chan_handle_t audio_tx_handle{nullptr}; i2s_chan_handle_t audio_rx_handle{nullptr}; i2s_std_config_t audio_std_cfg{}; diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index 621e82f32..a78200b08 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -1,3 +1,6 @@ +#include +#include + #include "m5stack-tab5.hpp" //////////////////////// @@ -6,6 +9,14 @@ namespace espp { +// Map a 0-100% microphone volume onto the ES7210's analog gain steps +// (GAIN_0DB .. GAIN_37_5DB) +static es7210_gain_value_t microphone_gain_from_volume(float volume) { + int step = static_cast(std::lround(volume / 100.0f * static_cast(GAIN_37_5DB))); + step = std::clamp(step, static_cast(GAIN_0DB), static_cast(GAIN_37_5DB)); + return static_cast(step); +} + bool M5StackTab5::initialize_audio(uint32_t sample_rate, const espp::Task::BaseConfig &task_config) { logger_.info("Initializing dual audio system (ES8388 + ES7210) at {} Hz", sample_rate); @@ -38,38 +49,14 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, logger_.info("Configuring I2S standard mode with sample rate {} Hz", sample_rate); ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_tx_handle, &audio_std_cfg)); - // Optional RX channel for recording (ES7210) - logger_.info("Creating I2S channel for recording (RX)"); - i2s_tdm_config_t tdm_cfg = { - .clk_cfg = - { - .sample_rate_hz = (uint32_t)48000, - .clk_src = I2S_CLK_SRC_DEFAULT, - .ext_clk_freq_hz = 0, - .mclk_multiple = I2S_MCLK_MULTIPLE_256, - .bclk_div = 8, - }, - .slot_cfg = {.data_bit_width = I2S_DATA_BIT_WIDTH_16BIT, - .slot_bit_width = I2S_SLOT_BIT_WIDTH_AUTO, - .slot_mode = I2S_SLOT_MODE_STEREO, - .slot_mask = (i2s_tdm_slot_mask_t)(I2S_TDM_SLOT0 | I2S_TDM_SLOT1 | - I2S_TDM_SLOT2 | I2S_TDM_SLOT3), - .ws_width = I2S_TDM_AUTO_WS_WIDTH, - .ws_pol = false, - .bit_shift = true, - .left_align = false, - .big_endian = false, - .bit_order_lsb = false, - .skip_mask = false, - .total_slot = I2S_TDM_AUTO_SLOT_NUM}, - .gpio_cfg = {.mclk = audio_mclk_io, - .bclk = audio_sclk_io, - .ws = audio_lrck_io, - .dout = audio_dsdin_io, // ES8388 DSDIN (playback data input) - .din = audio_asdout_io, // ES7210 ASDOUT (record data output) - .invert_flags = {.mclk_inv = false, .bclk_inv = false, .ws_inv = false}}, - }; - ESP_ERROR_CHECK(i2s_channel_init_tdm_mode(audio_rx_handle, &tdm_cfg)); + // RX channel for recording (ES7210). The ES7210 is configured below for + // standard I2S output (16-bit stereo: microphone 1 on the left slot, + // microphone 2 on the right), and in full-duplex mode the RX module shares + // the TX BCLK/WS, so the RX channel must use the same standard-mode + // configuration - a different frame geometry (e.g. multi-slot TDM) would + // not match the shared clock. + logger_.info("Configuring I2S channel for recording (RX)"); + ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_rx_handle, &audio_std_cfg)); // Stream buffers and task auto tx_buf_size = calc_audio_buffer_size(sample_rate); @@ -161,21 +148,48 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, if (es7210_adc_config_i2s(AUDIO_HAL_CODEC_MODE_ENCODE, &es7210_cfg.i2s_iface) != ESP_OK) { logger_.error("ES7210 I2S cfg failed"); } + // Enable the ES7210 digital high-pass filter on the ADC channels to strip + // the microphone DC offset. The espp es7210 driver leaves these registers + // at their reset value (HPF off), so the DC offset is amplified by the + // analog gain and rails the ADC to full scale - recordings come out as a + // near-constant DC level that the AC-coupled speaker plays as a click then + // silence. These are the driver's documented "quick setup" HPF values. + std::error_code hpf_ec; + es7210_i2c_device_->write_register(0x22, std::vector{0x0a}, hpf_ec); // ADC1/2 HPF1 + es7210_i2c_device_->write_register(0x23, std::vector{0x2a}, hpf_ec); // ADC1/2 HPF2 + es7210_i2c_device_->write_register(0x20, std::vector{0x0a}, hpf_ec); // ADC3/4 HPF2 + es7210_i2c_device_->write_register(0x21, std::vector{0x2a}, hpf_ec); // ADC3/4 HPF1 + if (hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", hpf_ec.message()); + } + // apply the stored microphone volume (the driver's init leaves the analog + // gain at its 0 dB minimum, which records very quietly) + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); es7210_adc_ctrl_state(AUDIO_HAL_CODEC_MODE_ENCODE, AUDIO_HAL_CTRL_START); - // now make the audio task - logger_.info("Creating audio task for playback and recording"); + // Create two independent tasks, matching the (well-behaved) esp-box audio + // path: one drains the playback stream buffer to the TX channel, and a + // separate task blocks on the RX channel to read the microphone. Keeping the + // record read off the playback task means the RX DMA ring is drained at + // exactly the rate samples arrive (rather than only between blocking TX + // writes), which the shared-task approach did irregularly - on this + // full-duplex controller that jitter degraded the recording and let the ring + // back up. + logger_.info("Creating audio playback and microphone tasks"); using namespace std::placeholders; audio_task_ = espp::Task::make_unique( {.callback = std::bind(&M5StackTab5::audio_task_callback, this, _1, _2, _3), .task_config = task_config}); + microphone_task_ = espp::Task::make_unique( + {.callback = std::bind(&M5StackTab5::microphone_task_callback, this, _1, _2, _3), + .task_config = task_config}); // Enable speaker output enable_audio(true); audio_initialized_ = true; - return audio_task_->start(); + return audio_task_->start() && microphone_task_->start(); } void M5StackTab5::enable_audio(bool enable) { @@ -200,26 +214,39 @@ void M5StackTab5::mute(bool mute) { bool M5StackTab5::is_muted() const { return mute_; } -void M5StackTab5::play_audio(const uint8_t *data, uint32_t num_bytes) { +size_t M5StackTab5::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!audio_initialized_ || !data || num_bytes == 0) { - return; + return 0; } - // Don't block here: append what fits into the stream buffer and return - // immediately. The audio task drains it to the I2S peripheral. Matches the - // esp-box / t-deck playback path. - xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); + // Don't block here: append what fits into the stream buffer and report how + // much was actually queued so callers can stream data larger than the + // buffer. The audio task drains it to the I2S peripheral. + return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); } -void M5StackTab5::play_audio(std::span data) { - play_audio(data.data(), data.size()); +size_t M5StackTab5::play_audio(std::span data) { + return play_audio(data.data(), data.size()); +} + +void M5StackTab5::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (audio_initialized_) { + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + } } +float M5StackTab5::microphone_volume() const { return mic_volume_; } + bool M5StackTab5::start_audio_recording( std::function callback) { if (!audio_initialized_) { logger_.error("Audio system not initialized"); return false; } + // The microphone task drains the RX ring continuously, so there is no stale + // backlog to flush here (and flushing would race that task on the same RX + // handle). Just install the callback and arm recording; the task starts + // forwarding samples on its next read. audio_rx_callback_ = callback; recording_ = true; logger_.info("Audio recording started"); @@ -249,14 +276,22 @@ bool M5StackTab5::audio_task_callback(std::mutex &m, std::condition_variable &cv i2s_channel_write(audio_tx_handle, tx_buf, buffer_size, NULL, portMAX_DELAY); } - // Recording: read from RX channel and invoke callback - if (recording_) { - size_t bytes_read = 0; - i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), &bytes_read, - 0); - if (bytes_read > 0 && audio_rx_callback_) { - audio_rx_callback_(audio_rx_buffer.data(), bytes_read); - } + return false; +} + +bool M5StackTab5::microphone_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + // Block until the RX channel has a buffer of microphone samples. This runs on + // its own task so the ES7210's RX DMA ring is always drained at the rate + // samples arrive - in full-duplex RX and TX share one I2S controller, so + // letting the ring back up perturbs the TX (pops the playback). Only forward + // the samples to the callback while a recording is in progress; otherwise the + // read still drains the ring and the data is discarded. + size_t bytes_read = 0; + auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), + &bytes_read, portMAX_DELAY); + if (err == ESP_OK && bytes_read > 0 && recording_ && audio_rx_callback_) { + audio_rx_callback_(audio_rx_buffer.data(), bytes_read); } return false; } diff --git a/components/m5stack-tab5/src/m5stack-tab5.cpp b/components/m5stack-tab5/src/m5stack-tab5.cpp index b7fb4e038..ae554cf46 100644 --- a/components/m5stack-tab5/src/m5stack-tab5.cpp +++ b/components/m5stack-tab5/src/m5stack-tab5.cpp @@ -21,7 +21,7 @@ bool M5StackTab5::initialize_io_expanders() { .device_address = 0x43, .timeout_ms = static_cast(internal_i2c_.config().timeout_ms), .scl_speed_hz = internal_i2c_.config().clk_speed, - .log_level = Logger::Verbosity::INFO, + .log_level = Logger::Verbosity::WARN, }, ec); if (!ioexp_0x43_i2c_device_) { @@ -33,7 +33,7 @@ bool M5StackTab5::initialize_io_expanders() { .device_address = 0x44, .timeout_ms = static_cast(internal_i2c_.config().timeout_ms), .scl_speed_hz = internal_i2c_.config().clk_speed, - .log_level = Logger::Verbosity::INFO, + .log_level = Logger::Verbosity::WARN, }, ec); if (!ioexp_0x44_i2c_device_) { diff --git a/components/smartpanlee-sc01-plus/example/README.md b/components/smartpanlee-sc01-plus/example/README.md index caca7ca07..069aac02d 100755 --- a/components/smartpanlee-sc01-plus/example/README.md +++ b/components/smartpanlee-sc01-plus/example/README.md @@ -10,12 +10,28 @@ BSP component on the Smart Panlee SC01 Plus touchscreen display module. - ST7796 display initialization over the ESP32-S3 8-bit parallel LCD bus - FT5x06 touch handling with LVGL integration - Foreground custom-drawn touch trail with bounded invalidation for smooth redraws -- I2S speaker playback with a bundled touch-click sound +- I2S speaker playback with a bundled touch-click sound, plus an on-screen + audio row (bottom-left) to play the sound, toggle mute, and adjust the + volume - Backlight brightness control - Optional microSD mounting and filesystem inspection - Published peripheral pin maps for I2S, RS-485, and external GPIOs - Rotation-aware UI layout with wrapped instructions that stay on-screen in portrait/landscape +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_SMARTPANLEE_SC01_PLUS_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. + ## Hardware Required - Smart Panlee SC01 Plus / WT32-SC01-PLUS compatible board diff --git a/components/smartpanlee-sc01-plus/example/main/gui.cpp b/components/smartpanlee-sc01-plus/example/main/gui.cpp index a86989878..c1119940e 100644 --- a/components/smartpanlee-sc01-plus/example/main/gui.cpp +++ b/components/smartpanlee-sc01-plus/example/main/gui.cpp @@ -4,9 +4,10 @@ void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); + init_tabview(); init_label(); init_buttons(); + init_audio_controls(); init_circle_layer(); // disable scrolling on the screen (so that it doesn't behave weirdly when // rotated and drawing with your finger) @@ -19,41 +20,46 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &board = espp::SmartPanleeSc01Plus::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, board.rotated_display_width(), board.rotated_display_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(8, 12, 24), 0); +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } void Gui::init_label() { - auto &board = espp::SmartPanleeSc01Plus::get(); - label_ = lv_label_create(lv_screen_active()); - lv_label_set_text(label_, ""); + label_ = lv_label_create(draw_tab_); lv_label_set_long_mode(label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(label_, lv_pct(75)); + lv_label_set_text(label_, ""); + lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 0, 0); lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_LEFT, 0); - // size / align the label for the current display width, leaving room on - // the right for the buttons - auto label_width = std::max(static_cast(board.rotated_display_width()) - 96, 120); - lv_obj_set_width(label_, label_width); - lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 16, 16); } void Gui::init_buttons() { - // a button in the top right which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); + // rotate / clear buttons in the top right of the Draw tab + rotate_button_ = lv_btn_create(draw_tab_); lv_obj_set_size(rotate_button_, 56, 56); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, -12, 12); + lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, 0, 0); lv_obj_t *rotate_label = lv_label_create(rotate_button_); lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); - // a button below the rotate button which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); + clear_button_ = lv_btn_create(draw_tab_); lv_obj_set_size(clear_button_, 56, 56); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, -12, 80); + lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 66); lv_obj_add_state(clear_button_, LV_STATE_CHECKED); // make the button red lv_obj_t *clear_label = lv_label_create(clear_button_); lv_label_set_text(clear_label, LV_SYMBOL_TRASH); @@ -61,11 +67,55 @@ void Gui::init_buttons() { lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); } -void Gui::init_circle_layer() { +void Gui::init_audio_controls() { + // the Audio tab: a column with the volume line, then play / mute / volume + // buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 12, 0); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 12, 0); + lv_obj_set_style_pad_row(row, 12, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&play_button_, LV_SYMBOL_PLAY}, + {&mute_button_, LV_SYMBOL_MUTE}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, + {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 56, 56); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } +} + +void Gui::update_audio_label() { auto &board = espp::SmartPanleeSc01Plus::get(); + lv_label_set_text_fmt(audio_label_, LV_SYMBOL_VOLUME_MAX " %d%%%s", + static_cast(board.volume()), board.is_muted() ? " (muted)" : ""); +} + +void Gui::init_circle_layer() { circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, board.rotated_display_width(), board.rotated_display_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); @@ -81,14 +131,7 @@ void Gui::update_layout() { auto &board = espp::SmartPanleeSc01Plus::get(); int width = board.rotated_display_width(); int height = board.rotated_display_height(); - lv_obj_set_size(background_, width, height); - // size / align the label for the new display width, leaving room on the - // right for the buttons - auto label_width = std::max(width - 96, 120); - lv_obj_set_width(label_, label_width); - lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 16, 16); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, -12, 12); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, -12, 80); + lv_obj_set_size(tabview_, width, height); lv_obj_set_size(circle_layer_, width, height); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_move_foreground(circle_layer_); @@ -99,6 +142,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -114,13 +161,51 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_PRESSED: gui->on_pressed(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_pressed(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &board = espp::SmartPanleeSc01Plus::get(); + if (target == play_button_) { + logger_.info("Play button pressed"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == mute_button_) { + board.mute(!board.is_muted()); + logger_.info("Muted: {}", board.is_muted()); + update_audio_label(); + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + board.volume(board.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", board.volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button pressed"); next_rotation(); @@ -130,6 +215,11 @@ void Gui::on_pressed(lv_event_t *e) { } } +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::set_label_text(std::string_view text) { std::lock_guard lock(mutex_); lv_label_set_text(label_, std::string(text).c_str()); diff --git a/components/smartpanlee-sc01-plus/example/main/gui.hpp b/components/smartpanlee-sc01-plus/example/main/gui.hpp index 8f751bcd4..e4113761c 100644 --- a/components/smartpanlee-sc01-plus/example/main/gui.hpp +++ b/components/smartpanlee-sc01-plus/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -23,13 +24,18 @@ /// member functions, keeping all UI logic inside the class. /// /// For this example the Gui shows: -/// * A wrapping info label (top-left) with instructions -/// * A button (top-right) which rotates the display through 0/90/180/270 -/// * A button (below the rotate button) which clears the circles drawn on -/// the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Draw" tab: instructions, plus the rotate and clear buttons. Touching +/// the screen while this tab is active draws circles (on a transparent +/// overlay) and plays a click. +/// * "Audio" tab: a play-sound button (wired to the example via a +/// callback), a mute toggle, and volume down / up buttons (acting +/// directly on the BSP), with a label showing the current volume. class Gui { public: + /// Callback invoked when the play-sound button is pressed + using audio_button_callback_t = std::function; /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -66,8 +72,18 @@ class Gui { /// re-aligning the UI to match. Thread-safe. void next_rotation(); + /// Whether the Draw tab is currently active (used by the example to only + /// draw circles / play clicks for touches on that tab). Thread-safe. + /// @return True if the Draw tab is the active tab + bool draw_page_active(); + + /// Set the callback invoked when the play-sound button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 44; struct Circle { int x{0}; @@ -80,11 +96,16 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); + void init_tabview(); void init_label(); void init_buttons(); + void init_audio_controls(); void init_circle_layer(); + // update the audio volume label from the BSP's current volume / mute + // state; called with the mutex held + void update_audio_label(); + // re-size / re-align the UI to fill the (possibly rotated) display void update_layout(); @@ -95,6 +116,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_pressed(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -105,18 +127,29 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *draw_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *label_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *mute_button_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + std::array circles_; size_t next_circle_index_{0}; size_t visible_circle_count_{0}; espp::Task update_task_{{.callback = [this](auto &m, auto &cv) { return update(m, cv); }, - .task_config = {.name = "gui", .stack_size_bytes = 6 * 1024}}}; + // NOTE: rendering the tabview (nested containers + flex layout) uses + // noticeably more stack than a flat UI; 6 KB overflows + .task_config = {.name = "gui", .stack_size_bytes = 12 * 1024}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp b/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp old mode 100755 new mode 100644 index 0a47e4e34..400183019 --- a/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp +++ b/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp @@ -67,11 +67,15 @@ extern "C" void app_main(void) { // thread-safe, so the touch callback below can call them directly. static Gui gui({.log_level = espp::Logger::Verbosity::INFO}); gui.set_label_text( - fmt::format("Smart Panlee SC01 Plus\n\nTouch the screen to draw and play a click.\nPress {} " - "to rotate.\nPress {} to clear.\nCheck serial output for SD card, pin, and " - "audio info.", + fmt::format("Smart Panlee SC01 Plus\n\nTouch the screen to draw and play a click.\nPress " + "{} to rotate.\nPress {} to clear.\nThe Audio tab plays the click sound and " + "adjusts / mutes the volume.\nCheck serial output for SD card, pin, and audio " + "info.", LV_SYMBOL_REFRESH, LV_SYMBOL_TRASH)); + // the play button on the audio row plays the same click sound as a touch + gui.set_play_callback([&]() { play_click(board); }); + // initialize the touchpad after the GUI exists so touch events can update // it immediately; each touch draws a circle and plays a click sound auto touch_callback = [&](const auto &touch) { @@ -79,12 +83,16 @@ extern "C" void app_main(void) { auto touchpad_data = board.touchpad_convert(touch); if (touchpad_data != previous_touchpad_data) { previous_touchpad_data = touchpad_data; - if (touchpad_data.num_touch_points > 0) { + if (touchpad_data.num_touch_points > 0 && gui.draw_page_active()) { play_click(board); gui.draw_circle(touchpad_data.x, touchpad_data.y, 10); } } }; + // NOTE: this example raises the BSP interrupt-task stack size via + // sdkconfig.defaults (CONFIG_SMARTPANLEE_SC01_PLUS_INTERRUPT_STACK_SIZE=8192); + // the touch controller is read from that task and its error-logging path + // needs more than the 4 KB BSP default. See the example README. if (!board.initialize_touch(touch_callback)) { logger.warn("Touch initialization did not complete cleanly"); } diff --git a/components/smartpanlee-sc01-plus/example/sdkconfig.defaults b/components/smartpanlee-sc01-plus/example/sdkconfig.defaults index 58f8b5d0f..3303bfb58 100755 --- a/components/smartpanlee-sc01-plus/example/sdkconfig.defaults +++ b/components/smartpanlee-sc01-plus/example/sdkconfig.defaults @@ -15,7 +15,10 @@ CONFIG_PARTITION_TABLE_CUSTOM=y CONFIG_PARTITION_TABLE_OFFSET=0x9000 CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" -CONFIG_SMARTPANLEE_SC01_PLUS_INTERRUPT_STACK_SIZE=4096 +# Raised from the 4 KB BSP default: the interrupt task runs the touch- +# controller I2C reads and, on a bus error, logs through espp's fmt +# color-print path, which needs more than 4 KB of stack. See the README. +CONFIG_SMARTPANLEE_SC01_PLUS_INTERRUPT_STACK_SIZE=8192 CONFIG_LV_DEF_REFR_PERIOD=16 CONFIG_LV_USE_THEME_DEFAULT=y diff --git a/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp b/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp index 0497d273f..d26243680 100755 --- a/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp +++ b/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp @@ -247,11 +247,17 @@ class SmartPanleeSc01Plus : public BaseComponent { float volume() const; /// Queue raw PCM audio bytes for playback. /// \param data Audio payload to play. - void play_audio(std::span data); + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full); stream longer data + /// by calling repeatedly, advancing by the returned count + size_t play_audio(std::span data); /// Queue raw PCM audio bytes for playback. /// \param data Pointer to PCM audio bytes. /// \param num_bytes Number of bytes to play. - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full); stream longer + /// data by calling repeatedly, advancing by the returned count + size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Initialize and mount the optional microSD card with default settings. /// \return True if the card was mounted successfully, false otherwise. diff --git a/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp b/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp index ee88b5b53..2991a20da 100755 --- a/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp +++ b/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp @@ -381,15 +381,17 @@ void SmartPanleeSc01Plus::volume(float volume) { volume_ = std::clamp(volume, 0. float SmartPanleeSc01Plus::volume() const { return volume_; } -void SmartPanleeSc01Plus::play_audio(std::span data) { - play_audio(data.data(), static_cast(data.size())); +size_t SmartPanleeSc01Plus::play_audio(std::span data) { + return play_audio(data.data(), static_cast(data.size())); } -void SmartPanleeSc01Plus::play_audio(const uint8_t *data, uint32_t num_bytes) { +size_t SmartPanleeSc01Plus::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!audio_initialized_ || !audio_tx_stream_ || !data || num_bytes == 0) { - return; + return 0; } - xStreamBufferSend(audio_tx_stream_, data, num_bytes, 0); + // non-blocking: the number of bytes actually queued is returned so callers + // can stream data larger than the buffer + return xStreamBufferSend(audio_tx_stream_, data, num_bytes, 0); } size_t SmartPanleeSc01Plus::rotated_display_width() const { diff --git a/components/t-deck/CMakeLists.txt b/components/t-deck/CMakeLists.txt index e0311d718..4265b1785 100644 --- a/components/t-deck/CMakeLists.txt +++ b/components/t-deck/CMakeLists.txt @@ -2,6 +2,6 @@ idf_component_register( INCLUDE_DIRS "include" SRC_DIRS "src" - REQUIRES driver esp_driver_i2s esp_driver_spi base_component display display_drivers fatfs i2c input_drivers interrupt gt911 spi task t_keyboard + REQUIRES driver esp_driver_i2s esp_driver_spi base_component codec display display_drivers fatfs i2c input_drivers interrupt gt911 spi task t_keyboard REQUIRED_IDF_TARGETS "esp32s3" ) diff --git a/components/t-deck/README.md b/components/t-deck/README.md index 09875a694..21245c7f3 100644 --- a/components/t-deck/README.md +++ b/components/t-deck/README.md @@ -15,6 +15,9 @@ subsystems. - GT911 capacitive touch input with LVGL integration helpers - T-Keyboard input and trackball pointer input - I2S speaker output with software volume / mute control +- Dual microphone recording through the ES7210 ADC (on its own I2S bus), + delivering 16-bit stereo frames to a callback with adjustable analog gain + via `microphone_volume()` - microSD mounting over SDSPI on the shared SPI host - Backlight brightness control and exposed peripheral helpers diff --git a/components/t-deck/example/README.md b/components/t-deck/example/README.md index 3c20445f6..ddd768edb 100644 --- a/components/t-deck/example/README.md +++ b/components/t-deck/example/README.md @@ -14,6 +14,24 @@ https://github.com/esp-cpp/espp/assets/213467/dc476c3d-dd9e-4b65-8c2d-9eda3ff3f3 ![image](https://github.com/esp-cpp/espp/assets/213467/4744d6ee-33bd-4907-8c58-3f3c2e5b7ba6) +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_TDECK_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. + +The example also enables `CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK`, which +turns any future task-stack overflow into an immediate, clearly-named panic +instead of silent heap corruption. + ## How to use example ### Hardware Required @@ -43,3 +61,17 @@ See the Getting Started Guide for full steps to configure and use ESP-IDF to bui - Trackball initialization and callback wiring - Optional uSD card mounting over SDSPI on the same SPI host - WAV playback through the onboard audio path +- Microphone recording and playback: the on-screen audio row (or the 'r' / + 'p' keys) records from the ES7210 into a PSRAM-preferred buffer and streams + it back through the mono speaker, with speaker / microphone volume buttons; + the measured effective capture rate is logged when a recording stops. + + Note on audio quality: the T-Deck's microphone is very low-sensitivity + (LilyGO's own firmware reads only ~200 counts for loud speech), and its + capture picks up random electrical impulse noise. The example therefore + post-processes each recording before playback: it de-glitches the impulse + noise (mark-and-interpolate), removes the DC offset, and applies an + RMS-normalized software makeup gain (the analog gain alone is not enough). + Only one microphone (MIC1) is used and mirrored to both channels for the + mono speaker. Even so, recordings are quiet and noisy - this is the limit + of the board's microphone hardware, not a configuration issue. diff --git a/components/t-deck/example/main/gui.cpp b/components/t-deck/example/main/gui.cpp index 7614e61bb..c7669b879 100644 --- a/components/t-deck/example/main/gui.cpp +++ b/components/t-deck/example/main/gui.cpp @@ -1,15 +1,14 @@ +#include +#include + #include "gui.hpp" void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); - init_label(); - init_buttons(); + init_tabview(); + init_draw_tab(); + init_audio_tab(); init_circle_layer(); - // disable scrolling on the screen (so that it doesn't behave weirdly when - // rotated and drawing with your finger) - lv_obj_set_scrollbar_mode(lv_screen_active(), LV_SCROLLBAR_MODE_OFF); - lv_obj_clear_flag(lv_screen_active(), LV_OBJ_FLAG_SCROLLABLE); } void Gui::deinit_ui() { @@ -17,53 +16,124 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &tdeck = espp::TDeck::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, tdeck.lcd_width(), tdeck.lcd_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(0, 0, 0), 0); +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } -void Gui::init_label() { - label_ = lv_label_create(lv_screen_active()); +void Gui::init_draw_tab() { + // instructions on the left, with the rotate / clear buttons on the right + label_ = lv_label_create(draw_tab_); + lv_label_set_long_mode(label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(label_, lv_pct(70)); lv_label_set_text(label_, ""); - lv_obj_align(label_, LV_ALIGN_CENTER, 0, 0); - lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_CENTER, 0); -} + lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_LEFT, 0); -void Gui::init_buttons() { - // a button in the top left which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); + rotate_button_ = lv_btn_create(draw_tab_); lv_obj_set_size(rotate_button_, 50, 50); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, 0, 0); lv_obj_t *rotate_label = lv_label_create(rotate_button_); lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); - // a button in the top right which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); + clear_button_ = lv_btn_create(draw_tab_); lv_obj_set_size(clear_button_, 50, 50); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 0); + lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 60); lv_obj_t *clear_label = lv_label_create(clear_button_); lv_label_set_text(clear_label, LV_SYMBOL_TRASH); lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); } -void Gui::init_circle_layer() { +void Gui::init_audio_tab() { + // a column: status line, volume line, then the buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 10, 0); + + audio_status_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_status_label_, lv_pct(100)); + lv_label_set_text(audio_status_label_, "Idle"); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 10, 0); + lv_obj_set_style_pad_row(row, 10, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&record_button_, LV_SYMBOL_AUDIO}, {&play_button_, LV_SYMBOL_PLAY}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + {&mic_down_button_, LV_SYMBOL_MINUS}, {&mic_up_button_, LV_SYMBOL_PLUS}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 44, 44); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } + // remember the record / play button labels so set_record_active / + // set_play_active can swap their symbols + record_button_label_ = lv_obj_get_child(record_button_, 0); + play_button_label_ = lv_obj_get_child(play_button_, 0); + // color the volume buttons so the speaker and microphone pairs are + // distinguishable from each other + lv_obj_set_style_bg_color(mic_down_button_, lv_palette_main(LV_PALETTE_TEAL), 0); + lv_obj_set_style_bg_color(mic_up_button_, lv_palette_main(LV_PALETTE_TEAL), 0); +} + +void Gui::update_audio_label() { auto &tdeck = espp::TDeck::get(); + lv_label_set_text_fmt(audio_label_, + "Speaker %d%% (" LV_SYMBOL_VOLUME_MID "/" LV_SYMBOL_VOLUME_MAX + ")\nMic %d%% (teal " LV_SYMBOL_MINUS "/" LV_SYMBOL_PLUS ")", + static_cast(tdeck.volume()), + static_cast(tdeck.microphone_volume())); +} + +void Gui::refresh_audio_label() { + std::lock_guard lock(mutex_); + update_audio_label(); +} + +void Gui::init_circle_layer() { + // a transparent, click-through overlay above the tabview which shows the + // touch trail (only populated while the Draw tab is active) circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, tdeck.lcd_width(), tdeck.lcd_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); lv_obj_set_style_bg_opa(circle_layer_, LV_OPA_TRANSP, 0); - lv_obj_set_style_border_width(circle_layer_, 0, 0); - lv_obj_set_style_outline_width(circle_layer_, 0, 0); - lv_obj_set_style_shadow_width(circle_layer_, 0, 0); lv_obj_add_event_cb(circle_layer_, draw_circle_layer, LV_EVENT_DRAW_MAIN, this); lv_obj_move_foreground(circle_layer_); } @@ -72,6 +142,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -87,13 +161,59 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_PRESSED: gui->on_pressed(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_pressed(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &tdeck = espp::TDeck::get(); + if (target == record_button_) { + logger_.info("Record button pressed"); + if (record_callback_) { + record_callback_(); + } + return; + } + if (target == play_button_) { + logger_.info("Play button pressed"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + tdeck.volume(tdeck.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", tdeck.volume()); + update_audio_label(); + return; + } + if (target == mic_down_button_ || target == mic_up_button_) { + float delta = target == mic_down_button_ ? -10.0f : 10.0f; + tdeck.microphone_volume(tdeck.microphone_volume() + delta); + logger_.info("Microphone volume: {:.0f}%", tdeck.microphone_volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button pressed"); next_rotation(); @@ -103,22 +223,48 @@ void Gui::on_pressed(lv_event_t *e) { } } +void Gui::set_record_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(record_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_AUDIO); + lv_obj_set_style_bg_color( + record_button_, active ? lv_palette_main(LV_PALETTE_RED) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_play_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(play_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_PLAY); + lv_obj_set_style_bg_color( + play_button_, active ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_audio_status(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(audio_status_label_, std::string(text).c_str()); +} + void Gui::set_label_text(std::string_view text) { std::lock_guard lock(mutex_); lv_label_set_text(label_, std::string(text).c_str()); } +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::next_rotation() { std::lock_guard lock(mutex_); - auto &tdeck = espp::TDeck::get(); clear_circles_impl(); auto rotation = lv_display_get_rotation(lv_display_get_default()); rotation = static_cast((static_cast(rotation) + 1) % 4); lv_display_set_rotation(lv_display_get_default(), rotation); // update the size of the screen-filling objects - lv_obj_set_size(background_, tdeck.rotated_display_width(), tdeck.rotated_display_height()); - lv_obj_align(label_, LV_ALIGN_CENTER, 0, 0); - lv_obj_set_size(circle_layer_, tdeck.rotated_display_width(), tdeck.rotated_display_height()); + int width = lv_display_get_horizontal_resolution(lv_display_get_default()); + int height = lv_display_get_vertical_resolution(lv_display_get_default()); + lv_obj_set_size(tabview_, width, height); + lv_obj_set_size(circle_layer_, width, height); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_invalidate(circle_layer_); } diff --git a/components/t-deck/example/main/gui.hpp b/components/t-deck/example/main/gui.hpp index 78cf787f5..ee01aa386 100644 --- a/components/t-deck/example/main/gui.hpp +++ b/components/t-deck/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -23,13 +24,19 @@ /// dispatched through a single static trampoline (event_callback) into /// member functions, keeping all UI logic inside the class. /// -/// For this example the Gui shows: -/// * A label with instructions -/// * A button (top-left) which rotates the display through 0/90/180/270 -/// * A button (top-right) which clears the circles drawn on the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Draw" tab: instructions, plus the rotate and clear buttons. Touching +/// the screen while this tab is active draws circles (on a transparent +/// overlay) and plays a click. +/// * "Audio" tab: record / play buttons (wired to the example via +/// callbacks) and speaker / microphone volume buttons (acting directly on +/// the BSP), with labels showing the audio state and current volumes. class Gui { public: + /// Callback invoked when the record / play buttons are pressed + using audio_button_callback_t = std::function; + /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -48,10 +55,15 @@ class Gui { deinit_ui(); } - /// Set the text of the main label. Thread-safe. + /// Set the instruction text shown on the Draw tab. Thread-safe. /// @param text The text to display void set_label_text(std::string_view text); + /// Whether the Draw tab is currently active (used by the example to only + /// draw circles / play clicks for touches on that tab). Thread-safe. + /// @return True if the Draw tab is the active tab + bool draw_page_active(); + /// Draw a circle at the given screen coordinates, replacing the oldest /// circle if the maximum number are already visible. Thread-safe. /// @param x The x coordinate (screen space) @@ -66,8 +78,38 @@ class Gui { /// re-aligning the UI to match. Thread-safe. void next_rotation(); + /// Set the callback invoked when the record button is pressed + /// @param callback The callback to invoke + void set_record_callback(audio_button_callback_t callback) { + record_callback_ = std::move(callback); + } + + /// Set the callback invoked when the play button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + + /// Show whether a recording is in progress (turns the record button red + /// and changes its symbol to stop). Thread-safe. + /// @param active True while recording + void set_record_active(bool active); + + /// Show whether a playback is in progress (turns the play button green and + /// changes its symbol to stop). Thread-safe. + /// @param active True while playing + void set_play_active(bool active); + + /// Set the status line on the Audio tab (e.g. "Recording...", "Mic + /// unavailable"). Thread-safe. + /// @param text The text to display + void set_audio_status(std::string_view text); + + /// Refresh the audio volume label from the BSP's current volumes (e.g. + /// after the keyboard shortcuts change the volume). Thread-safe. + void refresh_audio_label(); + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 40; struct Circle { int x{0}; @@ -80,11 +122,15 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); - void init_label(); - void init_buttons(); + void init_tabview(); + void init_draw_tab(); + void init_audio_tab(); void init_circle_layer(); + // update the audio volume label from the BSP's current volumes; called + // with the mutex held + void update_audio_label(); + // the LVGL update task: calls lv_task_handler() under the mutex bool update(std::mutex &m, std::condition_variable &cv); @@ -92,6 +138,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_pressed(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -102,18 +149,35 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *draw_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *label_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *record_button_{nullptr}; + lv_obj_t *record_button_label_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *play_button_label_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *mic_down_button_{nullptr}; + lv_obj_t *mic_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t record_callback_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + std::array circles_; size_t next_circle_index_{0}; size_t visible_circle_count_{0}; espp::Task update_task_{{.callback = [this](auto &m, auto &cv) { return update(m, cv); }, - .task_config = {.name = "gui", .stack_size_bytes = 6 * 1024}}}; + // NOTE: rendering the tabview (nested containers + flex layout) uses + // noticeably more stack than a flat UI; 6 KB overflows + .task_config = {.name = "gui", .stack_size_bytes = 12 * 1024}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/t-deck/example/main/t_deck_example.cpp b/components/t-deck/example/main/t_deck_example.cpp index c4881d947..081525a65 100644 --- a/components/t-deck/example/main/t_deck_example.cpp +++ b/components/t-deck/example/main/t_deck_example.cpp @@ -1,7 +1,15 @@ +#include +#include #include +#include +#include +#include #include #include +#include +#include + #include "t-deck.hpp" #include "gui.hpp" @@ -12,6 +20,38 @@ static std::vector audio_bytes; static bool load_audio(size_t &out_size, size_t &out_sample_rate); static void play_click(espp::TDeck &tdeck); +static void resample_click(uint32_t from_rate, uint32_t to_rate); + +// Run all audio at 16 kHz - the rate LilyGO's own T-Deck firmware uses for +// the ES7210 (the codec sounds clean there, and higher rates on this board +// produce a distorted / robotic capture). The speaker plays at the same rate +// so a recording plays back at the correct pitch with no runtime rate +// switching; the bundled 44.1 kHz click is resampled to 16 kHz at load. +static constexpr uint32_t AUDIO_SAMPLE_RATE_HZ = 16000; +static constexpr uint32_t MIC_SAMPLE_RATE_HZ = AUDIO_SAMPLE_RATE_HZ; + +// Audio recording state (written by the microphone callback, read/controlled +// from the GUI / keyboard callbacks and main loop). The recorded data is +// 16-bit interleaved stereo at the microphone sample rate. +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 2; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static int64_t recording_start_us = 0; +static int64_t recording_last_us = 0; +// number of bytes to discard at the start of a recording: the I2S RX DMA and +// the ES7210 emit a burst of settling garbage right after a recording begins, +// which otherwise plays back as a click of static at the front of the clip +static std::atomic warmup_bytes_remaining{0}; +// record / play toggles, shared by the on-screen buttons and the keyboard +// (assigned once the microphone is initialized) +static std::function toggle_record; +static std::function toggle_play; extern "C" void app_main(void) { espp::Logger logger({.tag = "T-Deck Example", .level = espp::Logger::Verbosity::INFO}); @@ -27,8 +67,11 @@ extern "C" void app_main(void) { if (!tdeck.initialize_sdcard(sdcard_config)) { logger.warn("Failed to initialize uSD card, there may not be a uSD card inserted!"); } - // initialize the sound - if (!tdeck.initialize_sound()) { + // initialize the sound at the example's fixed rate (see AUDIO_SAMPLE_RATE_HZ) + // so the speaker runs at that rate from the start - no runtime rate + // reconfiguration, which is where a speaker-vs-recording pitch mismatch + // could creep in + if (!tdeck.initialize_sound(AUDIO_SAMPLE_RATE_HZ)) { logger.error("Failed to initialize sound!"); return; } @@ -50,8 +93,10 @@ extern "C" void app_main(void) { // so the keyboard and touch callbacks below can call them directly. static Gui gui({.log_level = espp::Logger::Verbosity::INFO}); gui.set_label_text( - fmt::format("Touch the screen!\nPress the delete key or the {} button to clear " - "circles.\nPress the space key or the {} button to rotate the display.", + fmt::format("Touch the screen to draw!\nPress the delete key or the {} button to clear " + "circles.\nPress the space key or the {} button to rotate the display.\n" + "The Audio tab (or the 'r' / 'p' keys) records and plays back audio; " + "'n' / '$' / 'm' adjust / mute the speaker volume.", LV_SYMBOL_TRASH, LV_SYMBOL_REFRESH)); // initialize the Keyboard after the Gui exists so key presses can act on it @@ -76,11 +121,23 @@ extern "C" void app_main(void) { logger.info("Decreasing volume"); tdeck.volume(tdeck.volume() - 10.0f); logger.info("Volume: {}", tdeck.volume()); + gui.refresh_audio_label(); } else if (key == '$') { // '$' key will increase audio volume (right of 'm' key) logger.info("Increasing volume"); tdeck.volume(tdeck.volume() + 10.0f); logger.info("Volume: {}", tdeck.volume()); + gui.refresh_audio_label(); + } else if (key == 'r') { + // 'r' key toggles recording from the microphones + if (toggle_record) { + toggle_record(); + } + } else if (key == 'p') { + // 'p' key toggles playback of the recording + if (toggle_play) { + toggle_play(); + } } }; bool start_task = true; @@ -89,11 +146,17 @@ extern "C" void app_main(void) { return; } - // initialize the trackball + // initialize the trackball. This example uses the microphone, and per + // LilyGO's T-Deck documentation GPIO0 (the trackball's center / click + // button, shared with BOOT) is not available while the microphone is + // enabled - leaving it configured produces spurious interrupts that jitter + // the audio capture (a robotic / staticy recording). So initialize the + // trackball with the center button disabled; the four directional pins + // still work. auto trackball_callback = [&](const auto &trackball) { logger.debug("Trackball: {}", trackball); }; - if (!tdeck.initialize_trackball(trackball_callback)) { + if (!tdeck.initialize_trackball(trackball_callback, 10, /*enable_center_button=*/false)) { logger.error("Failed to initialize trackball!"); return; } @@ -110,13 +173,18 @@ extern "C" void app_main(void) { if (touchpad_data != previous_touchpad_data) { logger.info("Touch: {}", touchpad_data); previous_touchpad_data = touchpad_data; - // if there is a touch point, draw a circle and play a click sound - if (touchpad_data.num_touch_points > 0) { + // if there is a touch point on the Draw tab, draw a circle and play a + // click sound (touches on the other tabs go to their widgets) + if (touchpad_data.num_touch_points > 0 && gui.draw_page_active()) { play_click(tdeck); gui.draw_circle(touchpad_data.x, touchpad_data.y, 10); } } }; + // NOTE: this example raises the BSP interrupt-task stack size via + // sdkconfig.defaults (CONFIG_TDECK_INTERRUPT_STACK_SIZE=8192); the touch + // controller is read from that task and its error-logging path needs + // more than the 4 KB BSP default. See the example README. if (!tdeck.initialize_touch(touch_callback)) { logger.error("Failed to initialize touchpad!"); return; @@ -129,10 +197,13 @@ extern "C" void app_main(void) { logger.error("Failed to load audio file!"); return; } - logger.info("Loaded {} bytes of audio", wav_size); + logger.info("Loaded {} bytes of audio ({} Hz)", wav_size, wav_sample_rate); - logger.info("Setting audio sample rate to {} Hz", wav_sample_rate); - tdeck.audio_sample_rate(wav_sample_rate); + // Run the whole example at 16 kHz (see AUDIO_SAMPLE_RATE_HZ). The speaker + // was already initialized at that rate; the bundled click was decoded at + // its native rate, so resample it to 16 kHz once here so it plays at the + // right pitch through the 16 kHz speaker. + resample_click(static_cast(wav_sample_rate), AUDIO_SAMPLE_RATE_HZ); // unmute the audio and set the volume to 20% tdeck.mute(false); @@ -141,9 +212,267 @@ extern "C" void app_main(void) { // set the display brightness to be 75% tdeck.brightness(75.0f); - // now just loop forever + // Initialize the microphones (the ES7210 is on its own I2S bus, so pick + // the speaker's sample rate to make the recording directly playable) and + // buffer the recorded stereo frames; the recording auto-stops when the + // buffer is full and the main loop notices and updates the GUI + auto mic_callback = [](const uint8_t *data, size_t num_bytes) { + if (!recording) { + return; + } + // drop the warm-up (settling) bytes at the very start of the recording + size_t warm = warmup_bytes_remaining; + if (warm > 0) { + if (num_bytes <= warm) { + warmup_bytes_remaining = warm - num_bytes; + return; + } + data += warm; + num_bytes -= warm; + warmup_bytes_remaining = 0; + } + int64_t now = esp_timer_get_time(); + // stamp the true start of retained audio on the first kept sample so the + // measured capture rate excludes the warm-up period + if (recording_start_us == 0) { + recording_start_us = now; + } + size_t offset = recording_len; + size_t to_copy = std::min(num_bytes, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = now; + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + recording = false; + } + }; + // record at the ES7210's comfortable 16 kHz (see MIC_SAMPLE_RATE_HZ), which + // is independent of the speaker's rate (the ES7210 is on its own I2S bus) + bool have_mic = tdeck.initialize_microphone(mic_callback, MIC_SAMPLE_RATE_HZ); + if (have_mic) { + // The T-Deck's electret mics are genuinely low-sensitivity (LilyGO's own + // firmware reads only ~200 counts for loud speech next to the mic), so the + // analog stage is driven fairly hard here (~30 dB). Earlier this railed the + // ADC, but that was the mic's DC offset being amplified with the ES7210 + // high-pass filter disabled; the driver now enables the HPF, so the signal + // swings symmetrically around zero and this gain no longer saturates. The + // software auto-gain applied on stop (see the RMS normalization below) + // makes up whatever level remains; the mic +/- buttons adjust from here. + tdeck.microphone_volume(70.0f); + // allocate the recording buffer (16-bit interleaved stereo): prefer + // PSRAM, fall back to a couple of seconds in internal RAM + size_t bytes_per_second = tdeck.microphone_sample_rate() * 2 * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + logger.warn("Could not allocate a recording buffer; recording disabled"); + gui.set_audio_status("No recording buffer"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz stereo)", recording_capacity / 1024, + recording_capacity / bytes_per_second, tdeck.microphone_sample_rate()); + } + } else { + logger.warn("Could not initialize the microphone!"); + gui.set_audio_status("Mic unavailable (see log)"); + } + + // The record button / 'r' key toggles recording; the play button / 'p' key + // toggles playback of the recording (streamed to the speaker by the main + // loop) + toggle_record = [&]() { + if (!have_mic || recording_capacity == 0) { + logger.warn("Recording unavailable (no microphone / no buffer)"); + gui.set_audio_status("Mic unavailable (see log)"); + return; + } + if (recording) { + recording = false; // the main loop notices and logs the summary + } else { + playing = false; + gui.set_play_active(false); + recording_len = 0; + // discard the first ~250 ms of settling garbage; recording_start_us is + // stamped by the mic callback on the first retained sample + warmup_bytes_remaining = (tdeck.microphone_sample_rate() / 4) * 2 * sizeof(int16_t); + recording_start_us = 0; + recording_last_us = 0; + recording = true; + gui.set_record_active(true); + gui.set_audio_status("Recording..."); + } + }; + toggle_play = [&]() { + if (playing) { + playing = false; + gui.set_play_active(false); + gui.set_audio_status("Playback stopped"); + } else if (recording_len > 0) { + recording = false; + playing = true; + gui.set_play_active(true); + gui.set_audio_status("Playing..."); + } else { + logger.info("Nothing recorded yet; press the record button (or 'r') first"); + gui.set_audio_status("Nothing recorded yet"); + } + }; + gui.set_record_callback(toggle_record); + gui.set_play_callback(toggle_play); + + // Main loop: stream any active playback to the speaker and notice when a + // recording stops (either button press or the buffer filling up) + size_t play_offset = 0; + bool was_recording = false; while (true) { - std::this_thread::sleep_for(1s); + // feed the active playback in chunks, advancing by however much the + // speaker's stream buffer accepted + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_play_active(false); + gui.set_audio_status("Playback done"); + logger.info("Playback done"); + } else { + // Top up the speaker's stream buffer until it is full (play_audio + // returns less than requested once it can't accept more). A single + // chunk per loop leaves the buffer able to drain between iterations, + // which underruns and sounds like static / dropouts during a + // continuous playback. + while (play_offset < len) { + size_t chunk = std::min(len - play_offset, 16384); + size_t queued = tdeck.play_audio(recording_buffer + play_offset, chunk); + play_offset += queued; + if (queued < chunk) { + break; // stream buffer full for now + } + } + } + } else { + play_offset = 0; + } + // notice when the recording stopped (button press or buffer full) + bool now_recording = recording; + if (was_recording && !now_recording) { + gui.set_record_active(false); + gui.set_audio_status(fmt::format("Recorded {:.1f}s ({} plays)", + static_cast(recording_len) / + (tdeck.microphone_sample_rate() * 2 * sizeof(int16_t)), + LV_SYMBOL_PLAY)); + // report the measured capture rate: stereo frames recorded over the + // wall clock they took to arrive should match the nominal sample rate + size_t num_frames = recording_len / (2 * sizeof(int16_t)); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_frames / elapsed_s : 0.0f; + // Post-process the recording for playback on the T-Deck's MONO speaker. + // Use a single microphone (MIC1, the left slot - summing the two + // spatially separated mics comb-filters the sound) and mirror it to both + // stereo channels. The processing chain is: (1) de-glitch the electrical + // impulse noise, (2) remove the DC offset, (3) apply an RMS-normalized + // software makeup gain. The T-Deck mic is low-sensitivity (LilyGO's own + // firmware sees only ~200 counts for loud speech), so the makeup gain - + // not the analog stage - provides the loudness; RMS (rather than peak) + // sets it so a residual glitch cannot collapse the gain, and clipping + // catches any amplified outliers. + auto *samples = reinterpret_cast(recording_buffer); + // De-glitch: the ES7210 capture on this board carries random, bursty + // electrical impulse noise - roughly 0.3% of samples jump to |v| ~ 8000+ + // in runs of one to a few samples. (The de-spike frame gaps were measured + // to be non-periodic, i.e. line noise coupling into the mic / I2S rather + // than a framing or DMA-boundary artifact, so it cannot be removed by any + // codec register and has to be concealed here.) Detect each glitch by its + // large deviation from a local 7-point median (which tracks the real, + // slew-limited waveform, so legitimate speech is never flagged) and then + // linearly interpolate across each contiguous bad run from the good + // samples on either side. Interpolating a whole run conceals short bursts + // that a single-sample median replacement would leave behind. Done before + // the RMS/gain measurement so the glitches cannot inflate the makeup gain. + static constexpr int32_t kSpikeDelta = 2500; + size_t despiked = 0; + std::vector bad(num_frames, 0); + for (size_t i = 3; i + 3 < num_frames; i++) { + int32_t w[7]; + for (int k = 0; k < 7; k++) { + w[k] = samples[2 * (i - 3 + k)]; + } + for (int a = 1; a < 7; a++) { // insertion sort the 7-sample window + int32_t key = w[a]; + int b = a - 1; + while (b >= 0 && w[b] > key) { + w[b + 1] = w[b]; + b--; + } + w[b + 1] = key; + } + int32_t median = w[3]; + if (std::abs(static_cast(samples[2 * i]) - median) > kSpikeDelta) { + bad[i] = 1; + } + } + for (size_t i = 0; i < num_frames;) { + if (!bad[i]) { + i++; + continue; + } + size_t j = i; // [i, j) is a contiguous run of bad samples + while (j < num_frames && bad[j]) { + j++; + } + int32_t left = (i > 0) ? samples[2 * (i - 1)] : 0; + int32_t right = (j < num_frames) ? samples[2 * j] : left; + int32_t span = static_cast(j - i) + 1; + for (size_t k = i; k < j; k++) { + int32_t t = static_cast(k - i) + 1; + samples[2 * k] = static_cast(left + (right - left) * t / span); + } + despiked += j - i; + i = j; + } + int16_t min_l = 32767, max_l = -32768; + int64_t sum_l = 0, sum_sq = 0; + for (size_t i = 0; i < num_frames; i++) { + int16_t l = samples[2 * i]; // MIC1 (left) + min_l = std::min(min_l, l); + max_l = std::max(max_l, l); + sum_l += l; + } + int32_t dc_l = + num_frames ? static_cast(sum_l / static_cast(num_frames)) : 0; + for (size_t i = 0; i < num_frames; i++) { + int32_t v = samples[2 * i] - dc_l; + sum_sq += static_cast(v) * v; + } + double rms = num_frames ? std::sqrt(static_cast(sum_sq) / num_frames) : 0.0; + // target RMS ~4000 (about -18 dBFS) leaves headroom; cap the gain so a + // near-silent recording is not blown up into noise + static constexpr double target_rms = 4000.0; + double gain = rms > 1.0 ? std::clamp(target_rms / rms, 1.0, 64.0) : 1.0; + for (size_t i = 0; i < num_frames; i++) { + int32_t v = static_cast((samples[2 * i] - dc_l) * gain); + int16_t mono = static_cast(std::clamp(v, -32768, 32767)); + samples[2 * i] = mono; + samples[2 * i + 1] = mono; // mono: duplicate MIC1 to both channels + } + logger.info("Recorded {} frames in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal)", + num_frames, elapsed_s, effective_hz, tdeck.microphone_sample_rate()); + logger.info(" MIC1: raw min={} max={} dc={} rms={:.0f}; de-spiked {} samples; " + "applied software gain {:.1f}x", + min_l, max_l, dc_l, rms, despiked, gain); + } + was_recording = now_recording; + // tick faster while playing so the stream buffer is topped up well before + // it can drain (avoids underrun static); idle more slowly otherwise + std::this_thread::sleep_for(playing ? 10ms : 50ms); } //! [t-deck example] } @@ -187,3 +516,33 @@ static void play_click(espp::TDeck &tdeck) { offset += bytes_to_play; } } + +// Resample the loaded click (16-bit interleaved stereo) in place from +// from_rate to to_rate with linear interpolation, so it plays at the correct +// pitch through the speaker running at to_rate. +static void resample_click(uint32_t from_rate, uint32_t to_rate) { + if (from_rate == to_rate || audio_bytes.empty() || to_rate == 0) { + return; + } + const auto *in = reinterpret_cast(audio_bytes.data()); + size_t in_frames = audio_bytes.size() / (2 * sizeof(int16_t)); + if (in_frames < 2) { + return; + } + size_t out_frames = static_cast(static_cast(in_frames) * to_rate / from_rate); + std::vector out(out_frames * 2); + double step = static_cast(from_rate) / to_rate; + for (size_t j = 0; j < out_frames; ++j) { + double src = j * step; + size_t i0 = static_cast(src); + size_t i1 = std::min(i0 + 1, in_frames - 1); + double frac = src - i0; + for (int ch = 0; ch < 2; ++ch) { + double a = in[2 * i0 + ch]; + double b = in[2 * i1 + ch]; + out[2 * j + ch] = static_cast(a + (b - a) * frac); + } + } + audio_bytes.assign(reinterpret_cast(out.data()), + reinterpret_cast(out.data() + out.size())); +} diff --git a/components/t-deck/example/sdkconfig.defaults b/components/t-deck/example/sdkconfig.defaults index 100486ab5..996de4a5a 100644 --- a/components/t-deck/example/sdkconfig.defaults +++ b/components/t-deck/example/sdkconfig.defaults @@ -42,3 +42,17 @@ CONFIG_LV_USE_THEME_DEFAULT=y CONFIG_LV_THEME_DEFAULT_DARK=y CONFIG_LV_THEME_DEFAULT_GROW=y CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=30 + +# PSRAM (octal PSRAM on the WROOM-1 R8 / R16V modules); used for the audio +# recording buffer in the example +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y + +# catch task stack overflows immediately (debug watchpoint on the stack end) +# instead of letting them silently corrupt the heap +CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y +# The interrupt task (touch I2C) logs errors through the fmt color-print +# path, which needs more than the 4 KB BSP default; give it 8 KB here so a +# transient touch-bus error cannot overflow its stack. See the README. +CONFIG_TDECK_INTERRUPT_STACK_SIZE=8192 diff --git a/components/t-deck/include/t-deck.hpp b/components/t-deck/include/t-deck.hpp index 7d8a0e257..6410ae4df 100644 --- a/components/t-deck/include/t-deck.hpp +++ b/components/t-deck/include/t-deck.hpp @@ -21,6 +21,7 @@ #include #include "base_component.hpp" +#include "es7210.hpp" #include "gt911.hpp" #include "i2c.hpp" #include "interrupt.hpp" @@ -194,8 +195,15 @@ class TDeck : public BaseComponent { /// \see trackball_data() /// \see trackball_read() /// \see set_trackball_sensitivity() + /// \param enable_center_button Whether to configure the trackball's center + /// (click) button on GPIO0. Set this to false when the microphone is + /// in use: per LilyGO's documentation GPIO0 is not available while the + /// microphone is enabled, and leaving the button interrupt configured + /// on GPIO0 while recording produces a burst of spurious interrupts + /// that jitters the real-time audio capture. The four directional + /// quadrature pins are unaffected and still work. bool initialize_trackball(const trackball_callback_t &trackball_cb = nullptr, - int sensitivity = 10); + int sensitivity = 10, bool enable_center_button = true); /// Get the trackball /// \return A shared pointer to the trackball @@ -456,13 +464,64 @@ class TDeck : public BaseComponent { float volume() const; /// Play audio - /// \param data The audio data to play - void play_audio(const std::vector &data); + /// \param data The audio data to play (16-bit signed interleaved stereo) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(const std::vector &data); /// Play audio - /// \param data The audio data to play + /// \param data The audio data to play (16-bit signed interleaved stereo) /// \param num_bytes The number of bytes to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(const uint8_t *data, uint32_t num_bytes); + + ///////////////////////////////////////////////////////////////////////////// + // Microphone + ///////////////////////////////////////////////////////////////////////////// + + /// Alias for the microphone callback, called with recorded audio data + using microphone_callback_t = std::function; + + /// Initialize the microphones (the dual microphone array through the + /// ES7210 ADC, on its own I2S bus) and start delivering audio data to the + /// provided callback + /// \param callback The callback to call with recorded audio data: 16-bit + /// signed interleaved stereo (microphone 1 on the left slot, + /// microphone 2 on the right) at \p sample_rate + /// \param sample_rate The sample rate for the microphones, in Hz. The + /// ES7210 is on a separate I2S bus from the speaker amplifier, so + /// this is independent of the speaker's sample rate. + /// \param task_config The configuration for the microphone task + /// \return true if the microphone was successfully initialized, false + /// otherwise + /// \note The callback runs in the microphone task's context, so the task's + /// stack must be large enough for whatever the callback does with + /// the audio data + bool initialize_microphone(const microphone_callback_t &callback, uint32_t sample_rate = 16000, + const espp::Task::BaseConfig &task_config = {.name = "microphone", + .stack_size_bytes = 4096, + .priority = 10, + .core_id = 1}); + + /// Get the microphone sample rate + /// \return The microphone sample rate, in Hz + uint32_t microphone_sample_rate() const; + + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100), mapped onto the + /// ES7210 analog microphone gain range (0 dB - +37.5 dB) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100) + float microphone_volume() const; protected: TDeck(); @@ -472,6 +531,7 @@ class TDeck : public BaseComponent { void on_trackball_interrupt(const espp::Interrupt::Event &event); bool initialize_i2s(uint32_t default_audio_rate); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); // common: // internal i2c (touchscreen, keyboard) @@ -495,6 +555,7 @@ class TDeck : public BaseComponent { // // static constexpr gpio_num_t es7210_int_io = GPIO_NUM_17; static constexpr gpio_num_t dmic_clk_io = GPIO_NUM_17; + static constexpr auto mic_i2s_port = I2S_NUM_1; // Audio Out (MAX98357A) static constexpr auto i2s_port = I2S_NUM_0; @@ -670,5 +731,16 @@ class TDeck : public BaseComponent { std::vector audio_tx_buffer; StreamBufferHandle_t audio_tx_stream; i2s_std_config_t audio_std_cfg; + + // microphone (ES7210 on its own I2S bus) + std::shared_ptr> es7210_i2c_device_; + std::atomic microphone_initialized_{false}; + microphone_callback_t microphone_callback_{nullptr}; + std::unique_ptr microphone_task_{nullptr}; + i2s_chan_handle_t audio_rx_handle{nullptr}; + std::vector audio_rx_buffer; + std::atomic mic_sample_rate_{0}; + // microphone volume (percent), mapped onto the ES7210 analog gain range + std::atomic mic_volume_{70.0f}; }; // class TDeck } // namespace espp diff --git a/components/t-deck/src/audio.cpp b/components/t-deck/src/audio.cpp index cea2507fa..67545eeae 100644 --- a/components/t-deck/src/audio.cpp +++ b/components/t-deck/src/audio.cpp @@ -1,3 +1,10 @@ +#include +#include +#include +#include + +#include + #include "t-deck.hpp" using namespace espp; @@ -6,6 +13,14 @@ using namespace espp; // Audio Functions // //////////////////////// +// Map a 0-100% microphone volume onto the ES7210's analog gain steps +// (GAIN_0DB .. GAIN_37_5DB) +static es7210_gain_value_t microphone_gain_from_volume(float volume) { + int step = static_cast(std::lround(volume / 100.0f * static_cast(GAIN_37_5DB))); + step = std::clamp(step, static_cast(GAIN_0DB), static_cast(GAIN_37_5DB)); + return static_cast(step); +} + bool TDeck::initialize_i2s(uint32_t default_audio_rate) { logger_.info("initializing i2s driver"); logger_.debug("Using newer I2S standard"); @@ -147,9 +162,205 @@ void TDeck::audio_sample_rate(uint32_t sample_rate) { i2s_channel_enable(audio_tx_handle); } -void TDeck::play_audio(const std::vector &data) { play_audio(data.data(), data.size()); } +size_t TDeck::play_audio(const std::vector &data) { + return play_audio(data.data(), data.size()); +} -void TDeck::play_audio(const uint8_t *data, uint32_t num_bytes) { - // don't block here - xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); +size_t TDeck::play_audio(const uint8_t *data, uint32_t num_bytes) { + // don't block here: append what fits into the stream buffer and report how + // much was actually queued so callers can stream data larger than the + // buffer + return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); } + +////////////////////////// +// Microphone Functions // +////////////////////////// + +bool TDeck::initialize_microphone(const microphone_callback_t &callback, uint32_t sample_rate, + const espp::Task::BaseConfig &task_config) { + logger_.info("Initializing microphone"); + if (microphone_initialized_) { + logger_.warn("Microphone already initialized, not initializing again!"); + return false; + } + if (!callback) { + logger_.error("A callback is required to receive the recorded audio data"); + return false; + } + microphone_callback_ = callback; + + // The ES7210 has its own I2S bus (separate from the speaker amplifier), so + // make a receive-only master channel with its own clocks; the codec is a + // slave clocked from the MCLK / SCLK / LRCK we generate. + // + // The ES7210 outputs its four ADC channels as a 4-slot TDM frame (MIC1 -> + // slot 0, MIC2 -> slot 1, MIC3 -> slot 2, MIC4 -> slot 3), so read all four + // slots. The microphone task extracts the populated microphones (the T-Deck + // wires MIC1 and MIC3, i.e. slots 0 and 2 - one from each ADC pair) and + // presents them as 16-bit stereo. A 2-slot standard-I2S read would only see + // the first ADC pair (MIC1 + the unpopulated MIC2). + i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(mic_i2s_port, I2S_ROLE_MASTER); + ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, nullptr, &audio_rx_handle)); + i2s_tdm_config_t mic_tdm_cfg = { + .clk_cfg = {.sample_rate_hz = sample_rate, + .clk_src = I2S_CLK_SRC_DEFAULT, + .ext_clk_freq_hz = 0, + .mclk_multiple = I2S_MCLK_MULTIPLE_256, + // Let the driver derive BCLK from the slot count/width. The + // ES7210 coefficient table is programmed for MCLK = 256 x + // sample_rate (see MCLK_DIV_FRE in es7210.cpp), which + // mclk_multiple pins here. Forcing bclk_div (e.g. 8) makes + // the driver raise MCLK to satisfy the divider, so the codec + // receives an MCLK its coefficients were not computed for; + // its ADC decimation then drifts against LRCK and slips a + // frame every so often, which reads as sparse full-scale + // (0x8000) spikes on top of otherwise-clean audio. + .bclk_div = 0}, + .slot_cfg = {.data_bit_width = I2S_DATA_BIT_WIDTH_16BIT, + .slot_bit_width = I2S_SLOT_BIT_WIDTH_16BIT, + .slot_mode = I2S_SLOT_MODE_STEREO, + .slot_mask = static_cast(I2S_TDM_SLOT0 | I2S_TDM_SLOT1 | + I2S_TDM_SLOT2 | I2S_TDM_SLOT3), + .ws_width = I2S_TDM_AUTO_WS_WIDTH, + .ws_pol = false, + .bit_shift = true, + .left_align = false, + .big_endian = false, + .bit_order_lsb = false, + .skip_mask = false, + .total_slot = I2S_TDM_AUTO_SLOT_NUM}, + .gpio_cfg = {.mclk = es7210_mclk_io, + .bclk = es7210_sclk_io, + .ws = es7210_lrck_io, + .dout = GPIO_NUM_NC, + .din = es7210_sdout_io, + .invert_flags = {.mclk_inv = false, .bclk_inv = false, .ws_inv = false}}, + }; + ESP_ERROR_CHECK(i2s_channel_init_tdm_mode(audio_rx_handle, &mic_tdm_cfg)); + mic_sample_rate_ = sample_rate; + // the read buffer holds the raw 4-slot TDM frames (twice the size of the + // 2-channel stereo the callback receives) + audio_rx_buffer.resize(2 * calc_audio_buffer_size(sample_rate)); + ESP_ERROR_CHECK(i2s_channel_enable(audio_rx_handle)); + + // Configure the ES7210 ADC (microphone 1 -> left slot, microphone 2 -> + // right slot) now that its clocks are running + std::error_code ec; + es7210_i2c_device_ = internal_i2c_.add_device( + { + .device_address = 0x40, + .timeout_ms = static_cast(internal_i2c_.config().timeout_ms), + .scl_speed_hz = internal_i2c_.config().clk_speed, + .log_level = espp::Logger::Verbosity::WARN, + }, + ec); + if (!es7210_i2c_device_) { + logger_.error("Could not initialize ES7210 I2C device: {}", ec.message()); + i2s_channel_disable(audio_rx_handle); + i2s_del_channel(audio_rx_handle); + audio_rx_handle = nullptr; + return false; + } + set_es7210_write(espp::make_i2c_addressed_write(es7210_i2c_device_)); + set_es7210_read(espp::make_i2c_addressed_read_register(es7210_i2c_device_)); + + // Enable all four ADC channels so the ES7210 always emits a fixed 4-slot + // TDM frame ([MIC1, MIC2, MIC3, MIC4]). Enabling only a subset can make the + // codec pack fewer slots, which shifts the frame boundary against the fixed + // 4-slot read below and periodically lands the extraction on a floating, + // railing slot (heard as a robotic / static buzz on top of the audio). The + // T-Deck only populates MIC1 and MIC3, so the task keeps slots 0 and 2 and + // discards the unpopulated MIC2 / MIC4 slots. + es7210_mic_select(static_cast(ES7210_INPUT_MIC1 | ES7210_INPUT_MIC2 | + ES7210_INPUT_MIC3 | ES7210_INPUT_MIC4)); + + audio_hal_codec_config_t es7210_cfg{}; + es7210_cfg.codec_mode = AUDIO_HAL_CODEC_MODE_ENCODE; + es7210_cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; + es7210_cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; + es7210_cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; + es7210_cfg.i2s_iface.samples = AUDIO_HAL_16K_SAMPLES; + if (es7210_adc_init(&es7210_cfg) != ESP_OK) { + logger_.error("Could not initialize the ES7210 codec"); + i2s_channel_disable(audio_rx_handle); + i2s_del_channel(audio_rx_handle); + audio_rx_handle = nullptr; + return false; + } + if (es7210_adc_config_i2s(AUDIO_HAL_CODEC_MODE_ENCODE, &es7210_cfg.i2s_iface) != ESP_OK) { + logger_.error("ES7210 I2S config failed"); + } + // NOTE: do not write register 0x08 (master/slave & channels) here. The + // es7210 driver leaves it at its reset default in slave mode, which is + // correct for this board. Forcing 0x08 = 0x20 (as some MicroPython T-Deck + // ports do for their async PWM MCLK) reconfigures the channel/decimation + // path on our hardware-MCLK setup into a half-rate, every-other-frame-zero + // capture (MIC1 samples drop out and the glitch/de-spike rate rises), so it + // is deliberately left alone. + // Enable the ES7210 digital high-pass filter on the ADC channels to strip + // the microphone DC offset. The espp es7210 driver leaves these registers at + // their reset value (HPF off), so the (large, positive) DC offset from the + // T-Deck's electret mics is amplified by the analog gain and rails the ADC: + // captures come out all-positive with a big constant bias (e.g. MIC1 min=0 + // max=24672) that reads as a quiet-but-distorted / robotic recording, and at + // higher gain saturates to full scale. Removing the DC here lets the signal + // swing symmetrically around zero so the analog gain can be raised safely. + // These are the driver's documented "quick setup" HPF values, and are what + // the (working) esp-box microphone path uses. The T-Deck populates MIC1 and + // MIC3, one from each ADC pair, so both pairs' HPFs are enabled. + std::error_code hpf_ec; + es7210_i2c_device_->write_register(0x22, std::vector{0x0a}, hpf_ec); // ADC1/2 HPF1 + es7210_i2c_device_->write_register(0x23, std::vector{0x2a}, hpf_ec); // ADC1/2 HPF2 + es7210_i2c_device_->write_register(0x20, std::vector{0x0a}, hpf_ec); // ADC3/4 HPF2 + es7210_i2c_device_->write_register(0x21, std::vector{0x2a}, hpf_ec); // ADC3/4 HPF1 + if (hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", hpf_ec.message()); + } + // apply the stored microphone volume (the driver's init leaves the analog + // gain at its 0 dB minimum, which records very quietly) + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + es7210_adc_ctrl_state(AUDIO_HAL_CODEC_MODE_ENCODE, AUDIO_HAL_CTRL_START); + + using namespace std::placeholders; + microphone_task_ = espp::Task::make_unique({ + .callback = std::bind(&TDeck::microphone_task_callback, this, _1, _2, _3), + .task_config = task_config, + }); + + microphone_initialized_ = true; + + return microphone_task_->start(); +} + +uint32_t TDeck::microphone_sample_rate() const { return mic_sample_rate_; } + +bool TDeck::microphone_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + size_t bytes_read = 0; + auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), + &bytes_read, portMAX_DELAY); + if (err != ESP_OK || bytes_read == 0 || !microphone_callback_) { + return false; // don't stop the task + } + // audio_rx_buffer holds 4-slot TDM frames: [MIC1, MIC2, MIC3, MIC4]. Compact + // the populated microphones (MIC1 -> slot 0, MIC3 -> slot 2) down to 16-bit + // stereo in place (the output region is the front half, so this is safe). + auto *samples = reinterpret_cast(audio_rx_buffer.data()); + size_t num_frames = bytes_read / (4 * sizeof(int16_t)); + for (size_t i = 0; i < num_frames; i++) { + samples[2 * i] = samples[4 * i]; // MIC1 -> left + samples[2 * i + 1] = samples[4 * i + 2]; // MIC3 -> right + } + microphone_callback_(audio_rx_buffer.data(), num_frames * 2 * sizeof(int16_t)); + return false; // don't stop the task +} + +void TDeck::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (microphone_initialized_) { + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + } +} + +float TDeck::microphone_volume() const { return mic_volume_; } diff --git a/components/t-deck/src/t-deck.cpp b/components/t-deck/src/t-deck.cpp index cf80dc9fb..b1e230a66 100644 --- a/components/t-deck/src/t-deck.cpp +++ b/components/t-deck/src/t-deck.cpp @@ -88,7 +88,8 @@ std::shared_ptr TDeck::keyboard() const { return keyboard_; } // Trackball Functions // ///////////////////////// -bool TDeck::initialize_trackball(const TDeck::trackball_callback_t &trackball_cb, int sensitivity) { +bool TDeck::initialize_trackball(const TDeck::trackball_callback_t &trackball_cb, int sensitivity, + bool enable_center_button) { if (pointer_input_) { logger_.warn("Trackball already initialized, not initializing again!"); return false; @@ -102,12 +103,20 @@ bool TDeck::initialize_trackball(const TDeck::trackball_callback_t &trackball_cb // store the callback trackball_callback_ = trackball_cb; - // add the interrupts for the trackball + // add the interrupts for the trackball directional (quadrature) pins interrupts_.add_interrupt(trackball_up_interrupt_pin); interrupts_.add_interrupt(trackball_down_interrupt_pin); interrupts_.add_interrupt(trackball_left_interrupt_pin); interrupts_.add_interrupt(trackball_right_interrupt_pin); - interrupts_.add_interrupt(trackball_btn_interrupt_pin); + // the center (click) button is on GPIO0, which LilyGO documents as + // unavailable while the microphone is enabled; skip it when requested so + // microphone recording is not disrupted by spurious GPIO0 interrupts + if (enable_center_button) { + interrupts_.add_interrupt(trackball_btn_interrupt_pin); + } else { + logger_.info("Skipping trackball center button (GPIO0) - unavailable while the microphone " + "is in use"); + } // set the sensitivity set_trackball_sensitivity(sensitivity); From 6d4e1f85e8504faf086329dcaae6b055cb23e6bd Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 16:52:17 -0500 Subject: [PATCH 02/17] fix sa and comments --- components/esp-box/example/main/gui.cpp | 19 ++++++++++++++---- components/esp-box/example/main/gui.hpp | 4 ++++ components/esp-box/example/partitions.csv | 5 +++++ components/esp-box/example/sdkconfig.defaults | 6 ++++++ .../example/main/gui.cpp | 20 ++++++++++++++----- .../example/main/gui.hpp | 4 ++++ components/m5stack-tab5/example/main/gui.cpp | 20 ++++++++++++++----- components/m5stack-tab5/example/main/gui.hpp | 4 ++++ components/t-deck/example/main/gui.cpp | 20 ++++++++++++++----- components/t-deck/example/main/gui.hpp | 4 ++++ 10 files changed, 87 insertions(+), 19 deletions(-) create mode 100644 components/esp-box/example/partitions.csv diff --git a/components/esp-box/example/main/gui.cpp b/components/esp-box/example/main/gui.cpp index e477281c7..8a10ab668 100644 --- a/components/esp-box/example/main/gui.cpp +++ b/components/esp-box/example/main/gui.cpp @@ -144,10 +144,21 @@ void Gui::init_audio_tab() { void Gui::update_audio_label() { auto &box = espp::EspBox::get(); - lv_label_set_text_fmt(audio_label_, - "Speaker %d%% (" LV_SYMBOL_VOLUME_MID "/" LV_SYMBOL_VOLUME_MAX - ")\nMic %d%% (teal " LV_SYMBOL_MINUS "/" LV_SYMBOL_PLUS ")", - static_cast(box.volume()), static_cast(box.microphone_volume())); + int speaker_volume = static_cast(box.volume()); + int mic_volume = static_cast(box.microphone_volume()); + // this is called every GUI tick; only reformat the label when a value + // actually changes (lv_label_set_text_fmt formats a new string each call) + if (speaker_volume == last_speaker_volume_ && mic_volume == last_mic_volume_) { + return; + } + last_speaker_volume_ = speaker_volume; + last_mic_volume_ = mic_volume; + // Pass the LVGL symbols as %s arguments rather than concatenating them into + // the format-string literal: cppcheck cannot expand the LVGL symbol macros + // and flags the literal concatenation as an unknown macro. + lv_label_set_text_fmt(audio_label_, "Speaker %d%% (%s/%s)\nMic %d%% (teal %s/%s)", speaker_volume, + LV_SYMBOL_VOLUME_MID, LV_SYMBOL_VOLUME_MAX, mic_volume, LV_SYMBOL_MINUS, + LV_SYMBOL_PLUS); } void Gui::init_circle_layer() { diff --git a/components/esp-box/example/main/gui.hpp b/components/esp-box/example/main/gui.hpp index 7009aad03..88545590f 100644 --- a/components/esp-box/example/main/gui.hpp +++ b/components/esp-box/example/main/gui.hpp @@ -190,6 +190,10 @@ class Gui { lv_obj_t *mic_down_button_{nullptr}; lv_obj_t *mic_up_button_{nullptr}; lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_speaker_volume_{-1}; + int last_mic_volume_{-1}; lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; diff --git a/components/esp-box/example/partitions.csv b/components/esp-box/example/partitions.csv new file mode 100644 index 000000000..19d66ab8f --- /dev/null +++ b/components/esp-box/example/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0xa000, 0x6000 +phy_init, data, phy, , 0x1000 +factory, app, factory, , 6M +littlefs, data, littlefs, , 6M diff --git a/components/esp-box/example/sdkconfig.defaults b/components/esp-box/example/sdkconfig.defaults index 933ed1454..6b5707a51 100644 --- a/components/esp-box/example/sdkconfig.defaults +++ b/components/esp-box/example/sdkconfig.defaults @@ -10,6 +10,12 @@ CONFIG_ESPTOOLPY_FLASHSIZE="16MB" # over twice as fast as DIO CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +# Custom partition table: the tabbed UI + microphone example no longer fits the +# 1 MB default app partition, so use a 6 MB factory app (see partitions.csv). +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_OFFSET=0x9000 +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + # ESP32-specific # CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y diff --git a/components/esp32-p4-function-ev-board/example/main/gui.cpp b/components/esp32-p4-function-ev-board/example/main/gui.cpp index 4928eabc2..36d6fd751 100644 --- a/components/esp32-p4-function-ev-board/example/main/gui.cpp +++ b/components/esp32-p4-function-ev-board/example/main/gui.cpp @@ -128,11 +128,21 @@ void Gui::init_audio_controls() { void Gui::update_audio_label() { auto &board = espp::Esp32P4FunctionEvBoard::get(); - lv_label_set_text_fmt(audio_label_, - "Speaker %d%% (" LV_SYMBOL_VOLUME_MID "/" LV_SYMBOL_VOLUME_MAX - ")\nMic %d%% (teal " LV_SYMBOL_MINUS "/" LV_SYMBOL_PLUS ")", - static_cast(board.volume()), - static_cast(board.microphone_volume())); + int speaker_volume = static_cast(board.volume()); + int mic_volume = static_cast(board.microphone_volume()); + // this is called every GUI tick; only reformat the label when a value + // actually changes (lv_label_set_text_fmt formats a new string each call) + if (speaker_volume == last_speaker_volume_ && mic_volume == last_mic_volume_) { + return; + } + last_speaker_volume_ = speaker_volume; + last_mic_volume_ = mic_volume; + // Pass the LVGL symbols as %s arguments rather than concatenating them into + // the format-string literal: cppcheck cannot expand the LVGL symbol macros + // and flags the literal concatenation as an unknown macro. + lv_label_set_text_fmt(audio_label_, "Speaker %d%% (%s/%s)\nMic %d%% (teal %s/%s)", speaker_volume, + LV_SYMBOL_VOLUME_MID, LV_SYMBOL_VOLUME_MAX, mic_volume, LV_SYMBOL_MINUS, + LV_SYMBOL_PLUS); } void Gui::set_record_active(bool active) { diff --git a/components/esp32-p4-function-ev-board/example/main/gui.hpp b/components/esp32-p4-function-ev-board/example/main/gui.hpp index da8cc24fa..93501c259 100644 --- a/components/esp32-p4-function-ev-board/example/main/gui.hpp +++ b/components/esp32-p4-function-ev-board/example/main/gui.hpp @@ -165,6 +165,10 @@ class Gui { lv_obj_t *mic_down_button_{nullptr}; lv_obj_t *mic_up_button_{nullptr}; lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_speaker_volume_{-1}; + int last_mic_volume_{-1}; lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; diff --git a/components/m5stack-tab5/example/main/gui.cpp b/components/m5stack-tab5/example/main/gui.cpp index 0c883583f..04d2b7379 100644 --- a/components/m5stack-tab5/example/main/gui.cpp +++ b/components/m5stack-tab5/example/main/gui.cpp @@ -149,11 +149,21 @@ void Gui::init_audio_tab() { void Gui::update_audio_label() { auto &tab5 = espp::M5StackTab5::get(); - lv_label_set_text_fmt(audio_label_, - "Speaker %d%% (" LV_SYMBOL_VOLUME_MID "/" LV_SYMBOL_VOLUME_MAX - ")\nMic %d%% (teal " LV_SYMBOL_MINUS "/" LV_SYMBOL_PLUS ")", - static_cast(tab5.volume()), - static_cast(tab5.microphone_volume())); + int speaker_volume = static_cast(tab5.volume()); + int mic_volume = static_cast(tab5.microphone_volume()); + // this is called every GUI tick; only reformat the label when a value + // actually changes (lv_label_set_text_fmt formats a new string each call) + if (speaker_volume == last_speaker_volume_ && mic_volume == last_mic_volume_) { + return; + } + last_speaker_volume_ = speaker_volume; + last_mic_volume_ = mic_volume; + // Pass the LVGL symbols as %s arguments rather than concatenating them into + // the format-string literal: cppcheck cannot expand the LVGL symbol macros + // and flags the literal concatenation as an unknown macro. + lv_label_set_text_fmt(audio_label_, "Speaker %d%% (%s/%s)\nMic %d%% (teal %s/%s)", speaker_volume, + LV_SYMBOL_VOLUME_MID, LV_SYMBOL_VOLUME_MAX, mic_volume, LV_SYMBOL_MINUS, + LV_SYMBOL_PLUS); } void Gui::init_circle_layer() { diff --git a/components/m5stack-tab5/example/main/gui.hpp b/components/m5stack-tab5/example/main/gui.hpp index c5d182c37..099ee9511 100644 --- a/components/m5stack-tab5/example/main/gui.hpp +++ b/components/m5stack-tab5/example/main/gui.hpp @@ -197,6 +197,10 @@ class Gui { lv_obj_t *mic_down_button_{nullptr}; lv_obj_t *mic_up_button_{nullptr}; lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_speaker_volume_{-1}; + int last_mic_volume_{-1}; lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; diff --git a/components/t-deck/example/main/gui.cpp b/components/t-deck/example/main/gui.cpp index c7669b879..fc2602200 100644 --- a/components/t-deck/example/main/gui.cpp +++ b/components/t-deck/example/main/gui.cpp @@ -111,11 +111,21 @@ void Gui::init_audio_tab() { void Gui::update_audio_label() { auto &tdeck = espp::TDeck::get(); - lv_label_set_text_fmt(audio_label_, - "Speaker %d%% (" LV_SYMBOL_VOLUME_MID "/" LV_SYMBOL_VOLUME_MAX - ")\nMic %d%% (teal " LV_SYMBOL_MINUS "/" LV_SYMBOL_PLUS ")", - static_cast(tdeck.volume()), - static_cast(tdeck.microphone_volume())); + int speaker_volume = static_cast(tdeck.volume()); + int mic_volume = static_cast(tdeck.microphone_volume()); + // this is called every GUI tick; only reformat the label when a value + // actually changes (lv_label_set_text_fmt formats a new string each call) + if (speaker_volume == last_speaker_volume_ && mic_volume == last_mic_volume_) { + return; + } + last_speaker_volume_ = speaker_volume; + last_mic_volume_ = mic_volume; + // Pass the LVGL symbols as %s arguments rather than concatenating them into + // the format-string literal: cppcheck cannot expand the LVGL symbol macros + // and flags the literal concatenation as an unknown macro. + lv_label_set_text_fmt(audio_label_, "Speaker %d%% (%s/%s)\nMic %d%% (teal %s/%s)", speaker_volume, + LV_SYMBOL_VOLUME_MID, LV_SYMBOL_VOLUME_MAX, mic_volume, LV_SYMBOL_MINUS, + LV_SYMBOL_PLUS); } void Gui::refresh_audio_label() { diff --git a/components/t-deck/example/main/gui.hpp b/components/t-deck/example/main/gui.hpp index ee01aa386..e9ee325ec 100644 --- a/components/t-deck/example/main/gui.hpp +++ b/components/t-deck/example/main/gui.hpp @@ -164,6 +164,10 @@ class Gui { lv_obj_t *mic_down_button_{nullptr}; lv_obj_t *mic_up_button_{nullptr}; lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_speaker_volume_{-1}; + int last_mic_volume_{-1}; lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; From 1dfd931a880a87faaff39477bed021da52227397 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 16:53:15 -0500 Subject: [PATCH 03/17] address audio task name comment --- components/m5stack-tab5/src/audio.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index a78200b08..2b6c6d9cb 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -180,9 +180,15 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, audio_task_ = espp::Task::make_unique( {.callback = std::bind(&M5StackTab5::audio_task_callback, this, _1, _2, _3), .task_config = task_config}); + // give the microphone task a distinct name from the playback task (they share + // the rest of the caller's task config) so the two are easy to tell apart in + // logs and while debugging + auto mic_task_config = task_config; + mic_task_config.name = + task_config.name.empty() ? std::string("microphone") : task_config.name + " microphone"; microphone_task_ = espp::Task::make_unique( {.callback = std::bind(&M5StackTab5::microphone_task_callback, this, _1, _2, _3), - .task_config = task_config}); + .task_config = mic_task_config}); // Enable speaker output enable_audio(true); From 680db412cd435a7ec6723646d1e5a286ee6d317a Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 16:54:57 -0500 Subject: [PATCH 04/17] address doc comment --- components/m5stack-tab5/include/m5stack-tab5.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/m5stack-tab5/include/m5stack-tab5.hpp b/components/m5stack-tab5/include/m5stack-tab5.hpp index f42d07fe1..2794c6fec 100755 --- a/components/m5stack-tab5/include/m5stack-tab5.hpp +++ b/components/m5stack-tab5/include/m5stack-tab5.hpp @@ -333,7 +333,9 @@ class M5StackTab5 : public BaseComponent { /// signed interleaved stereo (ES7210 microphone 1 on the left slot, /// microphone 2 on the right) at audio_sample_rate() /// \return True if recording started successfully - /// \note The callback runs in the audio task's context + /// \note The callback runs in the dedicated microphone task's context (a + /// separate task from audio playback), so keep it quick and non- + /// blocking and size that task's stack accordingly bool start_audio_recording(std::function callback); /// Stop recording audio From 254b864327d8c5641559f0a007c5291ba495725f Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 19:20:32 -0500 Subject: [PATCH 05/17] address comments --- components/esp-box/src/audio.cpp | 43 +++++++++++- .../esp32-p4-function-ev-board/src/audio.cpp | 13 +++- .../m5stack-cardputer/src/microphone.cpp | 13 +++- components/m5stack-tab5/src/audio.cpp | 60 +++++++++++++++-- components/t-deck/src/audio.cpp | 66 +++++++++++++++---- 5 files changed, 169 insertions(+), 26 deletions(-) diff --git a/components/esp-box/src/audio.cpp b/components/esp-box/src/audio.cpp index bba4aacab..8825a87bd 100644 --- a/components/esp-box/src/audio.cpp +++ b/components/esp-box/src/audio.cpp @@ -17,6 +17,32 @@ static es7210_gain_value_t microphone_gain_from_volume(float volume) { return static_cast(step); } +// Map a PCM sample rate onto the codec's audio_hal sample-rate enum. The ES7210 +// coefficient table is selected by this enum, so it must match the rate the I2S +// clocks actually generate; an unsupported rate falls back to 48 kHz. +static audio_hal_iface_samples_t audio_hal_samples_from_rate(uint32_t rate) { + switch (rate) { + case 8000: + return AUDIO_HAL_08K_SAMPLES; + case 11025: + return AUDIO_HAL_11K_SAMPLES; + case 16000: + return AUDIO_HAL_16K_SAMPLES; + case 22050: + return AUDIO_HAL_22K_SAMPLES; + case 24000: + return AUDIO_HAL_24K_SAMPLES; + case 32000: + return AUDIO_HAL_32K_SAMPLES; + case 44100: + return AUDIO_HAL_44K_SAMPLES; + case 48000: + return AUDIO_HAL_48K_SAMPLES; + default: + return AUDIO_HAL_48K_SAMPLES; + } +} + bool EspBox::initialize_codec() { logger_.info("initializing codec"); @@ -278,7 +304,9 @@ bool EspBox::initialize_microphone(const microphone_callback_t &callback, es7210_cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; es7210_cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; es7210_cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; - es7210_cfg.i2s_iface.samples = AUDIO_HAL_48K_SAMPLES; + // configure the ES7210 for the rate the I2S clocks are actually running at + // (rather than assuming 48 kHz) so its coefficient table matches + es7210_cfg.i2s_iface.samples = audio_hal_samples_from_rate(audio_sample_rate()); if (es7210_adc_init(&es7210_cfg) != ESP_OK) { logger_.error("Could not initialize the ES7210 codec"); i2s_channel_disable(audio_rx_handle); @@ -320,12 +348,21 @@ bool EspBox::initialize_microphone(const microphone_callback_t &callback, bool EspBox::microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), - &bytes_read, portMAX_DELAY); + &bytes_read, pdMS_TO_TICKS(100)); if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { microphone_callback_(audio_rx_buffer.data(), bytes_read); } - return false; // don't stop the task + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running } void EspBox::microphone_volume(float volume) { diff --git a/components/esp32-p4-function-ev-board/src/audio.cpp b/components/esp32-p4-function-ev-board/src/audio.cpp index 2ed293167..2f50322d6 100644 --- a/components/esp32-p4-function-ev-board/src/audio.cpp +++ b/components/esp32-p4-function-ev-board/src/audio.cpp @@ -213,8 +213,11 @@ bool Esp32P4FunctionEvBoard::initialize_microphone(const microphone_callback_t & bool Esp32P4FunctionEvBoard::microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), - &bytes_read, portMAX_DELAY); + &bytes_read, pdMS_TO_TICKS(100)); if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { // compact the L,R word pairs down to mono in place, keeping the left // slot (the ES8311's ADC data) @@ -225,7 +228,13 @@ bool Esp32P4FunctionEvBoard::microphone_task_callback(std::mutex &m, std::condit } microphone_callback_(audio_rx_buffer.data(), num_frames * sizeof(int16_t)); } - return false; // don't stop the task + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running } void Esp32P4FunctionEvBoard::microphone_volume(float volume) { diff --git a/components/m5stack-cardputer/src/microphone.cpp b/components/m5stack-cardputer/src/microphone.cpp index e93c0384c..ab5ffdbdd 100644 --- a/components/m5stack-cardputer/src/microphone.cpp +++ b/components/m5stack-cardputer/src/microphone.cpp @@ -123,8 +123,11 @@ float M5StackCardputer::microphone_volume() const { return mic_volume_; } bool M5StackCardputer::microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), - &bytes_read, portMAX_DELAY); + &bytes_read, pdMS_TO_TICKS(100)); if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { if (mic_stereo_capture_) { // compact the L,R word pairs down to mono in place, keeping the left @@ -151,5 +154,11 @@ bool M5StackCardputer::microphone_task_callback(std::mutex &m, std::condition_va } microphone_callback_(audio_rx_buffer.data(), bytes_read); } - return false; // don't stop the task + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running } diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index 2b6c6d9cb..90b45501e 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -17,6 +17,32 @@ static es7210_gain_value_t microphone_gain_from_volume(float volume) { return static_cast(step); } +// Map a PCM sample rate onto the codec's audio_hal sample-rate enum. The ES7210 +// coefficient table is selected by this enum, so it must match the rate the I2S +// clocks actually generate; an unsupported rate falls back to 48 kHz. +static audio_hal_iface_samples_t audio_hal_samples_from_rate(uint32_t rate) { + switch (rate) { + case 8000: + return AUDIO_HAL_08K_SAMPLES; + case 11025: + return AUDIO_HAL_11K_SAMPLES; + case 16000: + return AUDIO_HAL_16K_SAMPLES; + case 22050: + return AUDIO_HAL_22K_SAMPLES; + case 24000: + return AUDIO_HAL_24K_SAMPLES; + case 32000: + return AUDIO_HAL_32K_SAMPLES; + case 44100: + return AUDIO_HAL_44K_SAMPLES; + case 48000: + return AUDIO_HAL_48K_SAMPLES; + default: + return AUDIO_HAL_48K_SAMPLES; + } +} + bool M5StackTab5::initialize_audio(uint32_t sample_rate, const espp::Task::BaseConfig &task_config) { logger_.info("Initializing dual audio system (ES8388 + ES7210) at {} Hz", sample_rate); @@ -139,7 +165,9 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, es7210_cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; es7210_cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; es7210_cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; - es7210_cfg.i2s_iface.samples = AUDIO_HAL_48K_SAMPLES; + // configure the ES7210 for the rate the I2S clocks are actually running at + // (rather than assuming 48 kHz) so its coefficient table matches + es7210_cfg.i2s_iface.samples = audio_hal_samples_from_rate(sample_rate); logger_.info("Initializing ES7210 codec for recording (ADC) at {} Hz", sample_rate); if (es7210_adc_init(&es7210_cfg) != ESP_OK) { logger_.error("ES7210 init failed"); @@ -193,9 +221,22 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, // Enable speaker output enable_audio(true); - audio_initialized_ = true; + // Start the tasks stepwise so a failure does not leave a half-initialized + // state: if the microphone task fails to start, stop the already-running + // playback task and report failure rather than leaving it running with + // audio_initialized_ set. + if (!audio_task_->start()) { + logger_.error("Failed to start the audio playback task"); + return false; + } + if (!microphone_task_->start()) { + logger_.error("Failed to start the microphone task"); + audio_task_->stop(); + return false; + } - return audio_task_->start() && microphone_task_->start(); + audio_initialized_ = true; + return true; } void M5StackTab5::enable_audio(bool enable) { @@ -294,12 +335,21 @@ bool M5StackTab5::microphone_task_callback(std::mutex &m, std::condition_variabl // the samples to the callback while a recording is in progress; otherwise the // read still drains the ring and the data is discarded. size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), - &bytes_read, portMAX_DELAY); + &bytes_read, pdMS_TO_TICKS(100)); if (err == ESP_OK && bytes_read > 0 && recording_ && audio_rx_callback_) { audio_rx_callback_(audio_rx_buffer.data(), bytes_read); } - return false; + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running } uint32_t M5StackTab5::audio_sample_rate() const { return audio_std_cfg.clk_cfg.sample_rate_hz; } diff --git a/components/t-deck/src/audio.cpp b/components/t-deck/src/audio.cpp index 67545eeae..30939c37e 100644 --- a/components/t-deck/src/audio.cpp +++ b/components/t-deck/src/audio.cpp @@ -21,6 +21,33 @@ static es7210_gain_value_t microphone_gain_from_volume(float volume) { return static_cast(step); } +// Map a PCM sample rate onto the codec's audio_hal sample-rate enum. The ES7210 +// coefficient table is selected by this enum, so it must match the rate the I2S +// clocks actually generate; an unsupported rate falls back to 16 kHz (the rate +// this board records at). +static audio_hal_iface_samples_t audio_hal_samples_from_rate(uint32_t rate) { + switch (rate) { + case 8000: + return AUDIO_HAL_08K_SAMPLES; + case 11025: + return AUDIO_HAL_11K_SAMPLES; + case 16000: + return AUDIO_HAL_16K_SAMPLES; + case 22050: + return AUDIO_HAL_22K_SAMPLES; + case 24000: + return AUDIO_HAL_24K_SAMPLES; + case 32000: + return AUDIO_HAL_32K_SAMPLES; + case 44100: + return AUDIO_HAL_44K_SAMPLES; + case 48000: + return AUDIO_HAL_48K_SAMPLES; + default: + return AUDIO_HAL_16K_SAMPLES; + } +} + bool TDeck::initialize_i2s(uint32_t default_audio_rate) { logger_.info("initializing i2s driver"); logger_.debug("Using newer I2S standard"); @@ -280,7 +307,9 @@ bool TDeck::initialize_microphone(const microphone_callback_t &callback, uint32_ es7210_cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; es7210_cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; es7210_cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; - es7210_cfg.i2s_iface.samples = AUDIO_HAL_16K_SAMPLES; + // configure the ES7210 for the requested capture rate (rather than assuming + // 16 kHz) so its coefficient table matches the I2S clocks + es7210_cfg.i2s_iface.samples = audio_hal_samples_from_rate(sample_rate); if (es7210_adc_init(&es7210_cfg) != ESP_OK) { logger_.error("Could not initialize the ES7210 codec"); i2s_channel_disable(audio_rx_handle); @@ -338,22 +367,31 @@ uint32_t TDeck::microphone_sample_rate() const { return mic_sample_rate_; } bool TDeck::microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), - &bytes_read, portMAX_DELAY); - if (err != ESP_OK || bytes_read == 0 || !microphone_callback_) { - return false; // don't stop the task + &bytes_read, pdMS_TO_TICKS(100)); + if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { + // audio_rx_buffer holds 4-slot TDM frames: [MIC1, MIC2, MIC3, MIC4]. + // Compact the populated microphones (MIC1 -> slot 0, MIC3 -> slot 2) down + // to 16-bit stereo in place (the output region is the front half, so this + // is safe). + auto *samples = reinterpret_cast(audio_rx_buffer.data()); + size_t num_frames = bytes_read / (4 * sizeof(int16_t)); + for (size_t i = 0; i < num_frames; i++) { + samples[2 * i] = samples[4 * i]; // MIC1 -> left + samples[2 * i + 1] = samples[4 * i + 2]; // MIC3 -> right + } + microphone_callback_(audio_rx_buffer.data(), num_frames * 2 * sizeof(int16_t)); } - // audio_rx_buffer holds 4-slot TDM frames: [MIC1, MIC2, MIC3, MIC4]. Compact - // the populated microphones (MIC1 -> slot 0, MIC3 -> slot 2) down to 16-bit - // stereo in place (the output region is the front half, so this is safe). - auto *samples = reinterpret_cast(audio_rx_buffer.data()); - size_t num_frames = bytes_read / (4 * sizeof(int16_t)); - for (size_t i = 0; i < num_frames; i++) { - samples[2 * i] = samples[4 * i]; // MIC1 -> left - samples[2 * i + 1] = samples[4 * i + 2]; // MIC3 -> right + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task } - microphone_callback_(audio_rx_buffer.data(), num_frames * 2 * sizeof(int16_t)); - return false; // don't stop the task + return false; // keep running } void TDeck::microphone_volume(float volume) { From 1094feb76789cb16d4d9d5e2bc9bf5c15f4f4f2f Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 20:01:43 -0500 Subject: [PATCH 06/17] improve api robustness --- components/esp-box/src/audio.cpp | 12 +++++++++--- components/m5stack-tab5/src/audio.cpp | 21 ++++++++++++++++----- components/t-deck/src/audio.cpp | 12 +++++++++--- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/components/esp-box/src/audio.cpp b/components/esp-box/src/audio.cpp index 8825a87bd..ac1d81de3 100644 --- a/components/esp-box/src/audio.cpp +++ b/components/esp-box/src/audio.cpp @@ -225,10 +225,16 @@ size_t EspBox::play_audio(const std::vector &data) { } size_t EspBox::play_audio(const uint8_t *data, uint32_t num_bytes) { + // guard against being called before initialize_sound() (audio_tx_stream is + // not valid until then) and against empty input + if (!sound_initialized_ || !data || num_bytes == 0) { + return 0; + } // don't block here: append what fits into the stream buffer and report how - // much was actually queued so callers can stream data larger than the - // buffer - return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); + // much was actually queued so callers can stream data larger than the buffer. + // This runs in task context, so use the non-ISR send with a 0 timeout (never + // blocks) rather than the FromISR variant. + return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); } ////////////////////////// diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index 90b45501e..491392d43 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -266,9 +266,11 @@ size_t M5StackTab5::play_audio(const uint8_t *data, uint32_t num_bytes) { return 0; } // Don't block here: append what fits into the stream buffer and report how - // much was actually queued so callers can stream data larger than the - // buffer. The audio task drains it to the I2S peripheral. - return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); + // much was actually queued so callers can stream data larger than the buffer. + // The audio task drains it to the I2S peripheral. This runs in task context, + // so use the non-ISR send with a 0 timeout (never blocks) rather than the + // FromISR variant. + return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); } size_t M5StackTab5::play_audio(std::span data) { @@ -290,10 +292,19 @@ bool M5StackTab5::start_audio_recording( logger_.error("Audio system not initialized"); return false; } + // Reject a start while a recording is already in progress: the microphone + // task reads audio_rx_callback_ whenever recording_ is true, so reassigning + // the callback here (without stopping first) would race that task. The caller + // must stop_audio_recording() before starting a new one. + if (recording_) { + logger_.warn("Recording already in progress; stop it before starting a new one"); + return false; + } // The microphone task drains the RX ring continuously, so there is no stale // backlog to flush here (and flushing would race that task on the same RX - // handle). Just install the callback and arm recording; the task starts - // forwarding samples on its next read. + // handle). Install the callback before arming recording_: the mic task only + // reads the callback once it observes recording_ == true, and the release/ + // acquire on that atomic publishes the callback write to it. audio_rx_callback_ = callback; recording_ = true; logger_.info("Audio recording started"); diff --git a/components/t-deck/src/audio.cpp b/components/t-deck/src/audio.cpp index 30939c37e..a360ca4dc 100644 --- a/components/t-deck/src/audio.cpp +++ b/components/t-deck/src/audio.cpp @@ -194,10 +194,16 @@ size_t TDeck::play_audio(const std::vector &data) { } size_t TDeck::play_audio(const uint8_t *data, uint32_t num_bytes) { + // guard against being called before initialize_sound() (audio_tx_stream is + // not valid until then) and against empty input + if (!sound_initialized_ || !data || num_bytes == 0) { + return 0; + } // don't block here: append what fits into the stream buffer and report how - // much was actually queued so callers can stream data larger than the - // buffer - return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); + // much was actually queued so callers can stream data larger than the buffer. + // This runs in task context, so use the non-ISR send with a 0 timeout (never + // blocks) rather than the FromISR variant. + return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); } ////////////////////////// From cc992c6b00dcbe7a1578864ce4b0e803382bf981 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 21:39:04 -0500 Subject: [PATCH 07/17] address comments and remove last dangling sendfromisr --- .../esp-box/example/main/esp_box_example.cpp | 4 ++-- components/esp-box/src/audio.cpp | 6 ++++++ .../main/esp32_p4_function_ev_board_example.cpp | 4 ++-- .../esp32-p4-function-ev-board/src/audio.cpp | 6 ++++++ components/m5stack-cardputer/src/audio.cpp | 17 +++++++++++++---- .../example/main/m5stack_tab5_example.cpp | 4 ++-- components/m5stack-tab5/src/audio.cpp | 6 ++++++ .../src/smartpanlee-sc01-plus.cpp | 6 ++++++ .../t-deck/example/main/t_deck_example.cpp | 4 ++-- components/t-deck/src/audio.cpp | 6 ++++++ 10 files changed, 51 insertions(+), 12 deletions(-) diff --git a/components/esp-box/example/main/esp_box_example.cpp b/components/esp-box/example/main/esp_box_example.cpp index 69aaffa07..493787218 100644 --- a/components/esp-box/example/main/esp_box_example.cpp +++ b/components/esp-box/example/main/esp_box_example.cpp @@ -36,8 +36,8 @@ static std::atomic recording_len{0}; static std::atomic playing{false}; // wall-clock bounds of the capture, for reporting the measured effective // sample rate (ordering is provided by the `recording` atomic) -static int64_t recording_start_us = 0; -static int64_t recording_last_us = 0; +static std::atomic recording_start_us{0}; +static std::atomic recording_last_us{0}; extern "C" void app_main(void) { espp::Logger logger({.tag = "ESP BOX Example", .level = espp::Logger::Verbosity::INFO}); diff --git a/components/esp-box/src/audio.cpp b/components/esp-box/src/audio.cpp index ac1d81de3..591a3e96c 100644 --- a/components/esp-box/src/audio.cpp +++ b/components/esp-box/src/audio.cpp @@ -230,6 +230,12 @@ size_t EspBox::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!sound_initialized_ || !data || num_bytes == 0) { return 0; } + // only enqueue whole 16-bit stereo frames (4 bytes) so a partial sample + // cannot shift the L/R framing or strand 1-3 bytes in the stream buffer + num_bytes -= num_bytes % 4; + if (num_bytes == 0) { + return 0; + } // don't block here: append what fits into the stream buffer and report how // much was actually queued so callers can stream data larger than the buffer. // This runs in task context, so use the non-ISR send with a 0 timeout (never diff --git a/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp b/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp index 40e65fd6e..6c0e6480d 100644 --- a/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp +++ b/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp @@ -52,8 +52,8 @@ static std::atomic recording_len{0}; static std::atomic playing{false}; // wall-clock bounds of the capture, for reporting the measured effective // sample rate (ordering is provided by the `recording` atomic) -static int64_t recording_start_us = 0; -static int64_t recording_last_us = 0; +static std::atomic recording_start_us{0}; +static std::atomic recording_last_us{0}; // Ping a target host a few times and log the result (uses the espp Ping helper). static void ping_target(espp::Logger &logger, const char *name, const std::string &ip) { diff --git a/components/esp32-p4-function-ev-board/src/audio.cpp b/components/esp32-p4-function-ev-board/src/audio.cpp index 2f50322d6..e48b968fe 100644 --- a/components/esp32-p4-function-ev-board/src/audio.cpp +++ b/components/esp32-p4-function-ev-board/src/audio.cpp @@ -131,6 +131,12 @@ size_t Esp32P4FunctionEvBoard::play_audio(const uint8_t *data, uint32_t num_byte if (!audio_initialized_ || !data || num_bytes == 0) { return 0; } + // only enqueue whole 16-bit samples (2 bytes; the TX slot is mono) so a + // partial sample cannot strand a byte in the stream buffer or shift framing + num_bytes -= num_bytes % 2; + if (num_bytes == 0) { + return 0; + } // Non-blocking: enqueue as much as currently fits in the TX stream buffer for // the audio task to drain to I2S. This never blocks the caller, so it is safe // to call from a touch callback / any task. The number of bytes actually diff --git a/components/m5stack-cardputer/src/audio.cpp b/components/m5stack-cardputer/src/audio.cpp index 8c59687d7..a63f4e538 100644 --- a/components/m5stack-cardputer/src/audio.cpp +++ b/components/m5stack-cardputer/src/audio.cpp @@ -249,10 +249,19 @@ size_t M5StackCardputer::play_audio(const std::vector &data) { } size_t M5StackCardputer::play_audio(const uint8_t *data, uint32_t num_bytes) { - if (!sound_initialized_) { + // guard against being called before initialize_sound() (audio_tx_stream is + // not valid until then) and against empty input + if (!sound_initialized_ || !data || num_bytes == 0) { + return 0; + } + // only enqueue whole 16-bit samples (2 bytes; the TX slot is mono) so a + // partial sample cannot strand a byte in the stream buffer or shift framing + num_bytes -= num_bytes % 2; + if (num_bytes == 0) { return 0; } - // don't block here; report how much was actually queued so callers can - // stream data larger than the buffer - return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); + // don't block here; report how much was actually queued so callers can stream + // data larger than the buffer. This runs in task context, so use the non-ISR + // send with a 0 timeout (never blocks) rather than the FromISR variant. + return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); } diff --git a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp index a19f5fa86..ef0288909 100644 --- a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp +++ b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp @@ -44,8 +44,8 @@ static std::atomic recording_len{0}; static std::atomic playing{false}; // wall-clock bounds of the capture, for reporting the measured effective // sample rate (ordering is provided by the `recording` atomic) -static int64_t recording_start_us = 0; -static int64_t recording_last_us = 0; +static std::atomic recording_start_us{0}; +static std::atomic recording_last_us{0}; extern "C" void app_main(void) { espp::Logger logger({.tag = "M5Stack Tab5 Example", .level = espp::Logger::Verbosity::INFO}); diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index 491392d43..088db8775 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -265,6 +265,12 @@ size_t M5StackTab5::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!audio_initialized_ || !data || num_bytes == 0) { return 0; } + // only enqueue whole 16-bit stereo frames (4 bytes) so a partial sample + // cannot shift the L/R framing or strand 1-3 bytes in the stream buffer + num_bytes -= num_bytes % 4; + if (num_bytes == 0) { + return 0; + } // Don't block here: append what fits into the stream buffer and report how // much was actually queued so callers can stream data larger than the buffer. // The audio task drains it to the I2S peripheral. This runs in task context, diff --git a/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp b/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp index 2991a20da..9e5926e72 100755 --- a/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp +++ b/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp @@ -389,6 +389,12 @@ size_t SmartPanleeSc01Plus::play_audio(const uint8_t *data, uint32_t num_bytes) if (!audio_initialized_ || !audio_tx_stream_ || !data || num_bytes == 0) { return 0; } + // only enqueue whole 16-bit stereo frames (4 bytes) so a partial sample + // cannot shift the L/R framing or strand 1-3 bytes in the stream buffer + num_bytes -= num_bytes % 4; + if (num_bytes == 0) { + return 0; + } // non-blocking: the number of bytes actually queued is returned so callers // can stream data larger than the buffer return xStreamBufferSend(audio_tx_stream_, data, num_bytes, 0); diff --git a/components/t-deck/example/main/t_deck_example.cpp b/components/t-deck/example/main/t_deck_example.cpp index 081525a65..75e22474b 100644 --- a/components/t-deck/example/main/t_deck_example.cpp +++ b/components/t-deck/example/main/t_deck_example.cpp @@ -42,8 +42,8 @@ static std::atomic recording_len{0}; static std::atomic playing{false}; // wall-clock bounds of the capture, for reporting the measured effective // sample rate (ordering is provided by the `recording` atomic) -static int64_t recording_start_us = 0; -static int64_t recording_last_us = 0; +static std::atomic recording_start_us{0}; +static std::atomic recording_last_us{0}; // number of bytes to discard at the start of a recording: the I2S RX DMA and // the ES7210 emit a burst of settling garbage right after a recording begins, // which otherwise plays back as a click of static at the front of the clip diff --git a/components/t-deck/src/audio.cpp b/components/t-deck/src/audio.cpp index a360ca4dc..a79db17ad 100644 --- a/components/t-deck/src/audio.cpp +++ b/components/t-deck/src/audio.cpp @@ -199,6 +199,12 @@ size_t TDeck::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!sound_initialized_ || !data || num_bytes == 0) { return 0; } + // only enqueue whole 16-bit stereo frames (4 bytes) so a partial sample + // cannot shift the L/R framing or strand 1-3 bytes in the stream buffer + num_bytes -= num_bytes % 4; + if (num_bytes == 0) { + return 0; + } // don't block here: append what fits into the stream buffer and report how // much was actually queued so callers can stream data larger than the buffer. // This runs in task context, so use the non-ISR send with a 0 timeout (never From 9ae5561fcc1abcf37eab2e862d643b59cc60215f Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 21:48:39 -0500 Subject: [PATCH 08/17] fix --- components/esp-box/example/main/esp_box_example.cpp | 2 +- .../example/main/esp32_p4_function_ev_board_example.cpp | 2 +- components/m5stack-tab5/example/main/m5stack_tab5_example.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/esp-box/example/main/esp_box_example.cpp b/components/esp-box/example/main/esp_box_example.cpp index 493787218..7b1b136fc 100644 --- a/components/esp-box/example/main/esp_box_example.cpp +++ b/components/esp-box/example/main/esp_box_example.cpp @@ -245,7 +245,7 @@ extern "C" void app_main(void) { gui.set_play_active(false); recording_len = 0; recording_start_us = esp_timer_get_time(); - recording_last_us = recording_start_us; + recording_last_us = recording_start_us.load(); recording = true; gui.set_record_active(true); gui.set_audio_status("Recording..."); diff --git a/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp b/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp index 6c0e6480d..1b50f2c36 100644 --- a/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp +++ b/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp @@ -311,7 +311,7 @@ extern "C" void app_main(void) { gui.set_play_active(false); recording_len = 0; recording_start_us = esp_timer_get_time(); - recording_last_us = recording_start_us; + recording_last_us = recording_start_us.load(); recording = true; gui.set_record_active(true); gui.set_audio_status("Recording..."); diff --git a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp index ef0288909..c34e55a7c 100644 --- a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp +++ b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp @@ -345,7 +345,7 @@ extern "C" void app_main(void) { gui.set_play_active(false); recording_len = 0; recording_start_us = esp_timer_get_time(); - recording_last_us = recording_start_us; + recording_last_us = recording_start_us.load(); recording = true; tab5.start_audio_recording(record_data_callback); gui.set_record_active(true); From 5cc553da21b8e71ee53538dab0abfc9fb087129c Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 22:31:53 -0500 Subject: [PATCH 09/17] address comments --- .../esp-box/example/main/esp_box_example.cpp | 18 +++++++++---- components/esp-box/src/audio.cpp | 21 ++++++++------- .../esp32-p4-function-ev-board/src/audio.cpp | 20 +++++++------- components/m5stack-cardputer/src/audio.cpp | 19 +++++++------ .../example/main/m5stack_tab5_example.cpp | 20 ++++++++++++-- components/m5stack-tab5/src/audio.cpp | 22 ++++++++------- .../main/smartpanlee_sc01_plus_example.cpp | 14 +++++++--- .../src/smartpanlee-sc01-plus.cpp | 18 ++++++++----- .../t-deck/example/main/t_deck_example.cpp | 19 +++++++++---- components/t-deck/include/t-deck.hpp | 5 ++-- components/t-deck/src/audio.cpp | 27 +++++++++++-------- 11 files changed, 132 insertions(+), 71 deletions(-) diff --git a/components/esp-box/example/main/esp_box_example.cpp b/components/esp-box/example/main/esp_box_example.cpp index 7b1b136fc..715232bd4 100644 --- a/components/esp-box/example/main/esp_box_example.cpp +++ b/components/esp-box/example/main/esp_box_example.cpp @@ -432,13 +432,21 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::EspBox &box) { - // use the box.play_audio() function to play a sound, breaking it into - // audio_buffer_size chunks + // Stream the click through play_audio() in audio_buffer_size chunks, + // advancing by however many bytes were actually queued: play_audio() enqueues + // only whole frames and may accept less than requested when the stream buffer + // is full, so advancing by the requested size would skip samples and misalign + // the rest of the clip. auto audio_buffer_size = box.audio_buffer_size(); size_t offset = 0; while (offset < audio_bytes.size()) { - size_t bytes_to_play = std::min(audio_buffer_size, audio_bytes.size() - offset); - box.play_audio(audio_bytes.data() + offset, bytes_to_play); - offset += bytes_to_play; + size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); + size_t queued = box.play_audio(audio_bytes.data() + offset, chunk); + if (queued == 0) { + // stream buffer momentarily full; let the audio task drain, then retry + std::this_thread::sleep_for(1ms); + continue; + } + offset += queued; } } diff --git a/components/esp-box/src/audio.cpp b/components/esp-box/src/audio.cpp index 591a3e96c..5195caeb2 100644 --- a/components/esp-box/src/audio.cpp +++ b/components/esp-box/src/audio.cpp @@ -230,17 +230,20 @@ size_t EspBox::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!sound_initialized_ || !data || num_bytes == 0) { return 0; } - // only enqueue whole 16-bit stereo frames (4 bytes) so a partial sample - // cannot shift the L/R framing or strand 1-3 bytes in the stream buffer - num_bytes -= num_bytes % 4; - if (num_bytes == 0) { + // Enqueue only whole 16-bit stereo frames (4 bytes) that actually fit: + // xStreamBufferSend can accept fewer bytes than requested when the buffer is + // nearly full, and a partial (non-frame-aligned) send would strand 1-3 bytes + // and corrupt the L/R framing of subsequent appends. Cap the request to the + // free space rounded down to a whole frame so the send is always + // frame-aligned. This runs in task context, so the non-ISR send with a 0 + // timeout never blocks. The number of bytes actually queued is returned so + // callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 4; + if (sendable == 0) { return 0; } - // don't block here: append what fits into the stream buffer and report how - // much was actually queued so callers can stream data larger than the buffer. - // This runs in task context, so use the non-ISR send with a 0 timeout (never - // blocks) rather than the FromISR variant. - return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); } ////////////////////////// diff --git a/components/esp32-p4-function-ev-board/src/audio.cpp b/components/esp32-p4-function-ev-board/src/audio.cpp index e48b968fe..f687c91df 100644 --- a/components/esp32-p4-function-ev-board/src/audio.cpp +++ b/components/esp32-p4-function-ev-board/src/audio.cpp @@ -131,17 +131,19 @@ size_t Esp32P4FunctionEvBoard::play_audio(const uint8_t *data, uint32_t num_byte if (!audio_initialized_ || !data || num_bytes == 0) { return 0; } - // only enqueue whole 16-bit samples (2 bytes; the TX slot is mono) so a - // partial sample cannot strand a byte in the stream buffer or shift framing - num_bytes -= num_bytes % 2; - if (num_bytes == 0) { + // Enqueue only whole 16-bit samples (2 bytes; the TX slot is mono) that + // actually fit: xStreamBufferSend can accept fewer bytes than requested when + // the buffer is nearly full, and a partial (odd) send would strand a byte and + // shift framing on subsequent appends. Cap the request to the free space + // rounded down to a whole sample. This runs in task context, so the non-ISR + // send with a 0 timeout never blocks; the number of bytes actually queued is + // returned so callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 2; + if (sendable == 0) { return 0; } - // Non-blocking: enqueue as much as currently fits in the TX stream buffer for - // the audio task to drain to I2S. This never blocks the caller, so it is safe - // to call from a touch callback / any task. The number of bytes actually - // queued is returned so callers can stream data larger than the buffer. - return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); } size_t Esp32P4FunctionEvBoard::play_audio(std::span data) { diff --git a/components/m5stack-cardputer/src/audio.cpp b/components/m5stack-cardputer/src/audio.cpp index a63f4e538..c89e4578c 100644 --- a/components/m5stack-cardputer/src/audio.cpp +++ b/components/m5stack-cardputer/src/audio.cpp @@ -254,14 +254,17 @@ size_t M5StackCardputer::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!sound_initialized_ || !data || num_bytes == 0) { return 0; } - // only enqueue whole 16-bit samples (2 bytes; the TX slot is mono) so a - // partial sample cannot strand a byte in the stream buffer or shift framing - num_bytes -= num_bytes % 2; - if (num_bytes == 0) { + // Enqueue only whole 16-bit samples (2 bytes; the TX slot is mono) that + // actually fit: xStreamBufferSend can accept fewer bytes than requested when + // the buffer is nearly full, and a partial (odd) send would strand a byte and + // shift framing on subsequent appends. Cap the request to the free space + // rounded down to a whole sample. This runs in task context, so the non-ISR + // send with a 0 timeout never blocks; the number of bytes actually queued is + // returned so callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 2; + if (sendable == 0) { return 0; } - // don't block here; report how much was actually queued so callers can stream - // data larger than the buffer. This runs in task context, so use the non-ISR - // send with a 0 timeout (never blocks) rather than the FromISR variant. - return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); } diff --git a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp index c34e55a7c..5dbe4fc26 100644 --- a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp +++ b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp @@ -576,7 +576,23 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::M5StackTab5 &tab5) { - if (audio_bytes.size() > 0) { - tab5.play_audio(audio_bytes); + // Stream the click through play_audio() in audio_buffer_size chunks, + // advancing by however many bytes were actually queued: play_audio() enqueues + // only whole frames and may accept less than requested when the stream buffer + // is full, so the whole click plays even if it is larger than that buffer. + if (audio_bytes.empty()) { + return; + } + auto audio_buffer_size = tab5.audio_buffer_size(); + size_t offset = 0; + while (offset < audio_bytes.size()) { + size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); + size_t queued = tab5.play_audio(audio_bytes.data() + offset, chunk); + if (queued == 0) { + // stream buffer momentarily full; let the audio task drain, then retry + std::this_thread::sleep_for(1ms); + continue; + } + offset += queued; } } diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index 088db8775..83f8312a4 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -265,18 +265,20 @@ size_t M5StackTab5::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!audio_initialized_ || !data || num_bytes == 0) { return 0; } - // only enqueue whole 16-bit stereo frames (4 bytes) so a partial sample - // cannot shift the L/R framing or strand 1-3 bytes in the stream buffer - num_bytes -= num_bytes % 4; - if (num_bytes == 0) { + // Enqueue only whole 16-bit stereo frames (4 bytes) that actually fit: + // xStreamBufferSend can accept fewer bytes than requested when the buffer is + // nearly full, and a partial (non-frame-aligned) send would strand 1-3 bytes + // and corrupt the L/R framing of subsequent appends. Cap the request to the + // free space rounded down to a whole frame so the send is always + // frame-aligned. This runs in task context, so the non-ISR send with a 0 + // timeout never blocks. The number of bytes actually queued is returned so + // callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 4; + if (sendable == 0) { return 0; } - // Don't block here: append what fits into the stream buffer and report how - // much was actually queued so callers can stream data larger than the buffer. - // The audio task drains it to the I2S peripheral. This runs in task context, - // so use the non-ISR send with a 0 timeout (never blocks) rather than the - // FromISR variant. - return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); } size_t M5StackTab5::play_audio(std::span data) { diff --git a/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp b/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp index 400183019..ed21d02be 100644 --- a/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp +++ b/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp @@ -143,10 +143,18 @@ static void play_click(espp::SmartPanleeSc01Plus &board) { return; } + // advance by however many bytes play_audio() actually queued: it enqueues + // only whole frames and may accept less than requested when the stream buffer + // is full, so advancing by the requested size would skip samples size_t offset = 0; while (offset < audio_bytes.size()) { - auto bytes_to_play = std::min(audio_buffer_size, audio_bytes.size() - offset); - board.play_audio(audio_bytes.data() + offset, static_cast(bytes_to_play)); - offset += bytes_to_play; + auto chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); + size_t queued = board.play_audio(audio_bytes.data() + offset, static_cast(chunk)); + if (queued == 0) { + // stream buffer momentarily full; let the audio task drain, then retry + std::this_thread::sleep_for(1ms); + continue; + } + offset += queued; } } diff --git a/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp b/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp index 9e5926e72..92148b8c2 100755 --- a/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp +++ b/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp @@ -389,15 +389,19 @@ size_t SmartPanleeSc01Plus::play_audio(const uint8_t *data, uint32_t num_bytes) if (!audio_initialized_ || !audio_tx_stream_ || !data || num_bytes == 0) { return 0; } - // only enqueue whole 16-bit stereo frames (4 bytes) so a partial sample - // cannot shift the L/R framing or strand 1-3 bytes in the stream buffer - num_bytes -= num_bytes % 4; - if (num_bytes == 0) { + // Enqueue only whole 16-bit stereo frames (4 bytes) that actually fit: + // xStreamBufferSend can accept fewer bytes than requested when the buffer is + // nearly full, and a partial (non-frame-aligned) send would strand 1-3 bytes + // and corrupt the L/R framing of subsequent appends. Cap the request to the + // free space rounded down to a whole frame so the send is always + // frame-aligned; the number of bytes actually queued is returned so callers + // can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream_)); + sendable -= sendable % 4; + if (sendable == 0) { return 0; } - // non-blocking: the number of bytes actually queued is returned so callers - // can stream data larger than the buffer - return xStreamBufferSend(audio_tx_stream_, data, num_bytes, 0); + return xStreamBufferSend(audio_tx_stream_, data, sendable, 0); } size_t SmartPanleeSc01Plus::rotated_display_width() const { diff --git a/components/t-deck/example/main/t_deck_example.cpp b/components/t-deck/example/main/t_deck_example.cpp index 75e22474b..3eecbd51d 100644 --- a/components/t-deck/example/main/t_deck_example.cpp +++ b/components/t-deck/example/main/t_deck_example.cpp @@ -506,14 +506,23 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::TDeck &tdeck) { - // use the tdeck.play_audio() function to play a sound, breaking it into - // audio_buffer_size chunks + // Stream the click through play_audio() in audio_buffer_size chunks, + // advancing by however many bytes were actually queued: play_audio() enqueues + // only whole frames and may accept less than requested when the stream buffer + // is full, so advancing by the requested size would skip samples and misalign + // the rest of the clip. (At 16 kHz audio_buffer_size is not a multiple of a + // 4-byte frame, so this matters here.) auto audio_buffer_size = tdeck.audio_buffer_size(); size_t offset = 0; while (offset < audio_bytes.size()) { - size_t bytes_to_play = std::min(audio_buffer_size, audio_bytes.size() - offset); - tdeck.play_audio(audio_bytes.data() + offset, bytes_to_play); - offset += bytes_to_play; + size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); + size_t queued = tdeck.play_audio(audio_bytes.data() + offset, chunk); + if (queued == 0) { + // stream buffer momentarily full; let the audio task drain, then retry + std::this_thread::sleep_for(1ms); + continue; + } + offset += queued; } } diff --git a/components/t-deck/include/t-deck.hpp b/components/t-deck/include/t-deck.hpp index 6410ae4df..e6177f5f5 100644 --- a/components/t-deck/include/t-deck.hpp +++ b/components/t-deck/include/t-deck.hpp @@ -493,8 +493,9 @@ class TDeck : public BaseComponent { /// ES7210 ADC, on its own I2S bus) and start delivering audio data to the /// provided callback /// \param callback The callback to call with recorded audio data: 16-bit - /// signed interleaved stereo (microphone 1 on the left slot, - /// microphone 2 on the right) at \p sample_rate + /// signed interleaved stereo. The two populated ES7210 microphones are + /// delivered as MIC1 (TDM slot 0) on the left channel and MIC3 (TDM + /// slot 2) on the right, at \p sample_rate /// \param sample_rate The sample rate for the microphones, in Hz. The /// ES7210 is on a separate I2S bus from the speaker amplifier, so /// this is independent of the speaker's sample rate. diff --git a/components/t-deck/src/audio.cpp b/components/t-deck/src/audio.cpp index a79db17ad..72992279f 100644 --- a/components/t-deck/src/audio.cpp +++ b/components/t-deck/src/audio.cpp @@ -199,17 +199,20 @@ size_t TDeck::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!sound_initialized_ || !data || num_bytes == 0) { return 0; } - // only enqueue whole 16-bit stereo frames (4 bytes) so a partial sample - // cannot shift the L/R framing or strand 1-3 bytes in the stream buffer - num_bytes -= num_bytes % 4; - if (num_bytes == 0) { + // Enqueue only whole 16-bit stereo frames (4 bytes) that actually fit: + // xStreamBufferSend can accept fewer bytes than requested when the buffer is + // nearly full, and a partial (non-frame-aligned) send would strand 1-3 bytes + // and corrupt the L/R framing of subsequent appends. Cap the request to the + // free space rounded down to a whole frame so the send is always + // frame-aligned. This runs in task context, so the non-ISR send with a 0 + // timeout never blocks. The number of bytes actually queued is returned so + // callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 4; + if (sendable == 0) { return 0; } - // don't block here: append what fits into the stream buffer and report how - // much was actually queued so callers can stream data larger than the buffer. - // This runs in task context, so use the non-ISR send with a 0 timeout (never - // blocks) rather than the FromISR variant. - return xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); } ////////////////////////// @@ -283,8 +286,10 @@ bool TDeck::initialize_microphone(const microphone_callback_t &callback, uint32_ audio_rx_buffer.resize(2 * calc_audio_buffer_size(sample_rate)); ESP_ERROR_CHECK(i2s_channel_enable(audio_rx_handle)); - // Configure the ES7210 ADC (microphone 1 -> left slot, microphone 2 -> - // right slot) now that its clocks are running + // Configure the ES7210 ADC now that its clocks are running. The ES7210 always + // emits a fixed 4-slot TDM frame; this board only populates MIC1 (slot 0) and + // MIC3 (slot 2), which the callback compacts to the left / right channels of + // the delivered 16-bit stereo. std::error_code ec; es7210_i2c_device_ = internal_i2c_.add_device( { From 48a7ae4e51bd9fb53cb3ed2e79c30460000e400b Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 22 Jul 2026 22:41:46 -0500 Subject: [PATCH 10/17] make sure the audio click is not blocking --- .../esp-box/example/main/esp_box_example.cpp | 20 ++++++++--------- .../example/main/m5stack_tab5_example.cpp | 19 ++++++++-------- .../main/smartpanlee_sc01_plus_example.cpp | 16 +++++++------- .../t-deck/example/main/t_deck_example.cpp | 22 +++++++++---------- 4 files changed, 39 insertions(+), 38 deletions(-) diff --git a/components/esp-box/example/main/esp_box_example.cpp b/components/esp-box/example/main/esp_box_example.cpp index 715232bd4..ec908eef3 100644 --- a/components/esp-box/example/main/esp_box_example.cpp +++ b/components/esp-box/example/main/esp_box_example.cpp @@ -432,21 +432,21 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::EspBox &box) { - // Stream the click through play_audio() in audio_buffer_size chunks, - // advancing by however many bytes were actually queued: play_audio() enqueues - // only whole frames and may accept less than requested when the stream buffer - // is full, so advancing by the requested size would skip samples and misalign - // the rest of the clip. + // Enqueue the click without blocking the caller (this runs in the touch + // callback). play_audio() enqueues only whole frames and returns how many + // bytes it took, so advance by that count (advancing by the requested size + // would skip samples). Stop as soon as the stream buffer is full rather than + // waiting for it to drain - blocking here would freeze the touch task for the + // whole click. The click comfortably fits in the stream buffer, so it plays + // in full in practice. auto audio_buffer_size = box.audio_buffer_size(); size_t offset = 0; while (offset < audio_bytes.size()) { size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); size_t queued = box.play_audio(audio_bytes.data() + offset, chunk); - if (queued == 0) { - // stream buffer momentarily full; let the audio task drain, then retry - std::this_thread::sleep_for(1ms); - continue; - } offset += queued; + if (queued < chunk) { + break; // stream buffer full for now; do not block the caller + } } } diff --git a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp index 5dbe4fc26..56ca14169 100644 --- a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp +++ b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp @@ -576,10 +576,13 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::M5StackTab5 &tab5) { - // Stream the click through play_audio() in audio_buffer_size chunks, - // advancing by however many bytes were actually queued: play_audio() enqueues - // only whole frames and may accept less than requested when the stream buffer - // is full, so the whole click plays even if it is larger than that buffer. + // Enqueue the click without blocking the caller (this runs in the touch + // callback). play_audio() enqueues only whole frames and returns how many + // bytes it took, so advance by that count (advancing by the requested size + // would skip samples). Stop as soon as the stream buffer is full rather than + // waiting for it to drain - blocking here would freeze the touch task for the + // whole click. The click comfortably fits in the stream buffer, so it plays + // in full in practice. if (audio_bytes.empty()) { return; } @@ -588,11 +591,9 @@ static void play_click(espp::M5StackTab5 &tab5) { while (offset < audio_bytes.size()) { size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); size_t queued = tab5.play_audio(audio_bytes.data() + offset, chunk); - if (queued == 0) { - // stream buffer momentarily full; let the audio task drain, then retry - std::this_thread::sleep_for(1ms); - continue; - } offset += queued; + if (queued < chunk) { + break; // stream buffer full for now; do not block the caller + } } } diff --git a/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp b/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp index ed21d02be..772bd65df 100644 --- a/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp +++ b/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp @@ -143,18 +143,18 @@ static void play_click(espp::SmartPanleeSc01Plus &board) { return; } - // advance by however many bytes play_audio() actually queued: it enqueues - // only whole frames and may accept less than requested when the stream buffer - // is full, so advancing by the requested size would skip samples + // Advance by however many bytes play_audio() actually queued (it enqueues + // only whole frames), not the requested size, so no samples are skipped. Stop + // as soon as the stream buffer is full rather than waiting for it to drain - + // this runs in the touch callback, so blocking would freeze the touch task + // for the whole click. The click comfortably fits in the stream buffer. size_t offset = 0; while (offset < audio_bytes.size()) { auto chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); size_t queued = board.play_audio(audio_bytes.data() + offset, static_cast(chunk)); - if (queued == 0) { - // stream buffer momentarily full; let the audio task drain, then retry - std::this_thread::sleep_for(1ms); - continue; - } offset += queued; + if (queued < chunk) { + break; // stream buffer full for now; do not block the caller + } } } diff --git a/components/t-deck/example/main/t_deck_example.cpp b/components/t-deck/example/main/t_deck_example.cpp index 3eecbd51d..d43090c5d 100644 --- a/components/t-deck/example/main/t_deck_example.cpp +++ b/components/t-deck/example/main/t_deck_example.cpp @@ -506,23 +506,23 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::TDeck &tdeck) { - // Stream the click through play_audio() in audio_buffer_size chunks, - // advancing by however many bytes were actually queued: play_audio() enqueues - // only whole frames and may accept less than requested when the stream buffer - // is full, so advancing by the requested size would skip samples and misalign - // the rest of the clip. (At 16 kHz audio_buffer_size is not a multiple of a - // 4-byte frame, so this matters here.) + // Enqueue the click without blocking the caller (this runs in the touch + // callback). play_audio() enqueues only whole frames and returns how many + // bytes it took, so advance by that count - advancing by the requested size + // would skip samples (at 16 kHz audio_buffer_size is not a multiple of a + // 4-byte frame, so this matters here). Stop as soon as the stream buffer is + // full rather than waiting for it to drain; blocking here would freeze the + // touch task for the whole click. The click comfortably fits in the stream + // buffer, so it plays in full in practice. auto audio_buffer_size = tdeck.audio_buffer_size(); size_t offset = 0; while (offset < audio_bytes.size()) { size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); size_t queued = tdeck.play_audio(audio_bytes.data() + offset, chunk); - if (queued == 0) { - // stream buffer momentarily full; let the audio task drain, then retry - std::this_thread::sleep_for(1ms); - continue; - } offset += queued; + if (queued < chunk) { + break; // stream buffer full for now; do not block the caller + } } } From 10f992a20c23d7b020174d84868a1fd59615eeca Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 23 Jul 2026 09:49:23 -0500 Subject: [PATCH 11/17] address comments --- components/esp-box/include/esp-box.hpp | 2 ++ components/esp-box/src/audio.cpp | 28 ++++++++++------- .../include/esp32-p4-function-ev-board.hpp | 2 ++ .../esp32-p4-function-ev-board/src/audio.cpp | 8 ++--- .../include/m5stack-cardputer.hpp | 2 ++ components/m5stack-cardputer/src/audio.cpp | 6 ++-- .../m5stack-tab5/include/m5stack-tab5.hpp | 2 ++ components/m5stack-tab5/src/audio.cpp | 28 ++++++++++------- .../example/main/gui.cpp | 5 +++- .../include/smartpanlee-sc01-plus.hpp | 2 ++ .../t-deck/example/main/t_deck_example.cpp | 10 +++++-- components/t-deck/include/t-deck.hpp | 2 ++ components/t-deck/src/audio.cpp | 30 ++++++++++++------- 13 files changed, 85 insertions(+), 42 deletions(-) diff --git a/components/esp-box/include/esp-box.hpp b/components/esp-box/include/esp-box.hpp index 0487cbfd1..38bc9ed26 100644 --- a/components/esp-box/include/esp-box.hpp +++ b/components/esp-box/include/esp-box.hpp @@ -329,6 +329,7 @@ class EspBox : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const std::vector &data); /// Play audio @@ -339,6 +340,7 @@ class EspBox : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const uint8_t *data, uint32_t num_bytes); ///////////////////////////////////////////////////////////////////////////// diff --git a/components/esp-box/src/audio.cpp b/components/esp-box/src/audio.cpp index 5195caeb2..5db533bff 100644 --- a/components/esp-box/src/audio.cpp +++ b/components/esp-box/src/audio.cpp @@ -159,9 +159,9 @@ void EspBox::enable_sound(bool enable) { gpio_set_level(sound_power_pin, enable) bool EspBox::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { // Queue the next I2S out frame to write - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); uint8_t *buffer = &audio_tx_buffer[0]; memset(buffer, 0, buffer_size); @@ -336,13 +336,21 @@ bool EspBox::initialize_microphone(const microphone_callback_t &callback, // analog gain and rails the ADC to full scale - recordings come out as a // near-constant DC level that the AC-coupled speaker plays as a click then // silence. These are the driver's documented "quick setup" HPF values. - std::error_code hpf_ec; - es7210_i2c_device_->write_register(0x22, std::vector{0x0a}, hpf_ec); // ADC1/2 HPF1 - es7210_i2c_device_->write_register(0x23, std::vector{0x2a}, hpf_ec); // ADC1/2 HPF2 - es7210_i2c_device_->write_register(0x20, std::vector{0x0a}, hpf_ec); // ADC3/4 HPF2 - es7210_i2c_device_->write_register(0x21, std::vector{0x2a}, hpf_ec); // ADC3/4 HPF1 - if (hpf_ec) { - logger_.warn("Could not enable the ES7210 high-pass filter: {}", hpf_ec.message()); + // write_register clears the error code on success, so a later successful + // write would mask an earlier failure; remember the first failure explicitly. + std::error_code hpf_ec, first_hpf_ec; + auto write_hpf = [&](uint8_t reg, uint8_t val) { + es7210_i2c_device_->write_register(reg, std::vector{val}, hpf_ec); + if (hpf_ec && !first_hpf_ec) { + first_hpf_ec = hpf_ec; + } + }; + write_hpf(0x22, 0x0a); // ADC1/2 HPF1 + write_hpf(0x23, 0x2a); // ADC1/2 HPF2 + write_hpf(0x20, 0x0a); // ADC3/4 HPF2 + write_hpf(0x21, 0x2a); // ADC3/4 HPF1 + if (first_hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", first_hpf_ec.message()); } // apply the stored microphone volume (the driver's init leaves the analog // gain at its 0 dB minimum, which records very quietly) diff --git a/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp b/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp index fd4a8f148..e6383ed98 100644 --- a/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp +++ b/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp @@ -277,6 +277,7 @@ class Esp32P4FunctionEvBoard : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Play audio data @@ -286,6 +287,7 @@ class Esp32P4FunctionEvBoard : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(std::span data); ///////////////////////////////////////////////////////////////////////////// diff --git a/components/esp32-p4-function-ev-board/src/audio.cpp b/components/esp32-p4-function-ev-board/src/audio.cpp index f687c91df..f015075ff 100644 --- a/components/esp32-p4-function-ev-board/src/audio.cpp +++ b/components/esp32-p4-function-ev-board/src/audio.cpp @@ -256,12 +256,12 @@ float Esp32P4FunctionEvBoard::microphone_volume() const { return mic_volume_; } bool Esp32P4FunctionEvBoard::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); // only ever hand whole 16-bit samples to I2S; a partial sample would shift // the framing of everything after it - available &= ~static_cast(1); + available &= ~static_cast(1); uint8_t *tx_buf = audio_tx_buffer.data(); memset(tx_buf, 0, buffer_size); if (available == 0) { diff --git a/components/m5stack-cardputer/include/m5stack-cardputer.hpp b/components/m5stack-cardputer/include/m5stack-cardputer.hpp index 5478fe434..7a1936ed3 100644 --- a/components/m5stack-cardputer/include/m5stack-cardputer.hpp +++ b/components/m5stack-cardputer/include/m5stack-cardputer.hpp @@ -415,6 +415,7 @@ class M5StackCardputer : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const std::vector &data); /// Play the audio data @@ -425,6 +426,7 @@ class M5StackCardputer : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const uint8_t *data, uint32_t num_bytes); ///////////////////////////////////////////////////////////////////////////// diff --git a/components/m5stack-cardputer/src/audio.cpp b/components/m5stack-cardputer/src/audio.cpp index c89e4578c..d664c7452 100644 --- a/components/m5stack-cardputer/src/audio.cpp +++ b/components/m5stack-cardputer/src/audio.cpp @@ -180,9 +180,9 @@ bool M5StackCardputer::initialize_sound(uint32_t default_audio_rate, bool M5StackCardputer::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { // Queue the next I2S out frame to write - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); uint8_t *buffer = &audio_tx_buffer[0]; memset(buffer, 0, buffer_size); diff --git a/components/m5stack-tab5/include/m5stack-tab5.hpp b/components/m5stack-tab5/include/m5stack-tab5.hpp index 2794c6fec..e9c34c7c9 100755 --- a/components/m5stack-tab5/include/m5stack-tab5.hpp +++ b/components/m5stack-tab5/include/m5stack-tab5.hpp @@ -317,6 +317,7 @@ class M5StackTab5 : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Play audio data @@ -326,6 +327,7 @@ class M5StackTab5 : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(std::span data); /// Start recording audio diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index 83f8312a4..17ec63e1e 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -182,13 +182,21 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, // analog gain and rails the ADC to full scale - recordings come out as a // near-constant DC level that the AC-coupled speaker plays as a click then // silence. These are the driver's documented "quick setup" HPF values. - std::error_code hpf_ec; - es7210_i2c_device_->write_register(0x22, std::vector{0x0a}, hpf_ec); // ADC1/2 HPF1 - es7210_i2c_device_->write_register(0x23, std::vector{0x2a}, hpf_ec); // ADC1/2 HPF2 - es7210_i2c_device_->write_register(0x20, std::vector{0x0a}, hpf_ec); // ADC3/4 HPF2 - es7210_i2c_device_->write_register(0x21, std::vector{0x2a}, hpf_ec); // ADC3/4 HPF1 - if (hpf_ec) { - logger_.warn("Could not enable the ES7210 high-pass filter: {}", hpf_ec.message()); + // write_register clears the error code on success, so a later successful + // write would mask an earlier failure; remember the first failure explicitly. + std::error_code hpf_ec, first_hpf_ec; + auto write_hpf = [&](uint8_t reg, uint8_t val) { + es7210_i2c_device_->write_register(reg, std::vector{val}, hpf_ec); + if (hpf_ec && !first_hpf_ec) { + first_hpf_ec = hpf_ec; + } + }; + write_hpf(0x22, 0x0a); // ADC1/2 HPF1 + write_hpf(0x23, 0x2a); // ADC1/2 HPF2 + write_hpf(0x20, 0x0a); // ADC3/4 HPF2 + write_hpf(0x21, 0x2a); // ADC3/4 HPF1 + if (first_hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", first_hpf_ec.message()); } // apply the stored microphone volume (the driver's init leaves the analog // gain at its 0 dB minimum, which records very quietly) @@ -330,9 +338,9 @@ bool M5StackTab5::audio_task_callback(std::mutex &m, std::condition_variable &cv // Queue the next I2S out frame to write (matches the esp-box / t-deck path): // always write a full, frame-aligned buffer with the queued samples zero- // padded up to buffer_size so the I2S DMA is fed at a constant cadence. - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); uint8_t *tx_buf = audio_tx_buffer.data(); memset(tx_buf, 0, buffer_size); if (available == 0) { diff --git a/components/smartpanlee-sc01-plus/example/main/gui.cpp b/components/smartpanlee-sc01-plus/example/main/gui.cpp index c1119940e..d11dc84ae 100644 --- a/components/smartpanlee-sc01-plus/example/main/gui.cpp +++ b/components/smartpanlee-sc01-plus/example/main/gui.cpp @@ -107,7 +107,10 @@ void Gui::init_audio_controls() { void Gui::update_audio_label() { auto &board = espp::SmartPanleeSc01Plus::get(); - lv_label_set_text_fmt(audio_label_, LV_SYMBOL_VOLUME_MAX " %d%%%s", + // pass the LVGL symbol as a %s argument rather than concatenating it into the + // format-string literal: cppcheck cannot expand the LVGL symbol macros and + // flags the literal concatenation as an unknown macro + lv_label_set_text_fmt(audio_label_, "%s %d%%%s", LV_SYMBOL_VOLUME_MAX, static_cast(board.volume()), board.is_muted() ? " (muted)" : ""); } diff --git a/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp b/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp index d26243680..98fd5f706 100755 --- a/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp +++ b/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp @@ -250,6 +250,7 @@ class SmartPanleeSc01Plus : public BaseComponent { /// \return The number of bytes actually queued (may be less than the data /// size if the internal stream buffer is full); stream longer data /// by calling repeatedly, advancing by the returned count + /// \note Must be called from task context, not from an ISR. size_t play_audio(std::span data); /// Queue raw PCM audio bytes for playback. /// \param data Pointer to PCM audio bytes. @@ -257,6 +258,7 @@ class SmartPanleeSc01Plus : public BaseComponent { /// \return The number of bytes actually queued (may be less than \p /// num_bytes if the internal stream buffer is full); stream longer /// data by calling repeatedly, advancing by the returned count + /// \note Must be called from task context, not from an ISR. size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Initialize and mount the optional microSD card with default settings. diff --git a/components/t-deck/example/main/t_deck_example.cpp b/components/t-deck/example/main/t_deck_example.cpp index d43090c5d..d2aa6aa12 100644 --- a/components/t-deck/example/main/t_deck_example.cpp +++ b/components/t-deck/example/main/t_deck_example.cpp @@ -533,11 +533,15 @@ static void resample_click(uint32_t from_rate, uint32_t to_rate) { if (from_rate == to_rate || audio_bytes.empty() || to_rate == 0) { return; } - const auto *in = reinterpret_cast(audio_bytes.data()); size_t in_frames = audio_bytes.size() / (2 * sizeof(int16_t)); if (in_frames < 2) { return; } + // Copy the samples into a properly-aligned int16_t buffer: audio_bytes is a + // byte vector, so reinterpreting its storage as int16_t* and dereferencing it + // would be unaligned / undefined behavior. + std::vector in(in_frames * 2); + std::memcpy(in.data(), audio_bytes.data(), in.size() * sizeof(int16_t)); size_t out_frames = static_cast(static_cast(in_frames) * to_rate / from_rate); std::vector out(out_frames * 2); double step = static_cast(from_rate) / to_rate; @@ -552,6 +556,6 @@ static void resample_click(uint32_t from_rate, uint32_t to_rate) { out[2 * j + ch] = static_cast(a + (b - a) * frac); } } - audio_bytes.assign(reinterpret_cast(out.data()), - reinterpret_cast(out.data() + out.size())); + audio_bytes.resize(out.size() * sizeof(int16_t)); + std::memcpy(audio_bytes.data(), out.data(), audio_bytes.size()); } diff --git a/components/t-deck/include/t-deck.hpp b/components/t-deck/include/t-deck.hpp index e6177f5f5..25999741b 100644 --- a/components/t-deck/include/t-deck.hpp +++ b/components/t-deck/include/t-deck.hpp @@ -470,6 +470,7 @@ class TDeck : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const std::vector &data); /// Play audio @@ -480,6 +481,7 @@ class TDeck : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const uint8_t *data, uint32_t num_bytes); ///////////////////////////////////////////////////////////////////////////// diff --git a/components/t-deck/src/audio.cpp b/components/t-deck/src/audio.cpp index 72992279f..b2dfeed0e 100644 --- a/components/t-deck/src/audio.cpp +++ b/components/t-deck/src/audio.cpp @@ -133,9 +133,9 @@ bool TDeck::initialize_sound(uint32_t default_audio_rate, bool TDeck::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { // Queue the next I2S out frame to write - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); uint8_t *buffer = &audio_tx_buffer[0]; memset(buffer, 0, buffer_size); @@ -152,7 +152,7 @@ bool TDeck::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool // NOTE: we are actually using int16_t samples, so we need to adjust the // pointers to the buffer to the correct positions. int16_t *ptr = reinterpret_cast(buffer); - for (int i = 0; i < (available / 2); ++i) { + for (size_t i = 0; i < (available / 2); ++i) { int32_t sample = (ptr[i] * volume_) / 100.0f; ptr[i] = static_cast(std::clamp(sample, INT16_MIN, INT16_MAX)); } @@ -355,13 +355,21 @@ bool TDeck::initialize_microphone(const microphone_callback_t &callback, uint32_ // These are the driver's documented "quick setup" HPF values, and are what // the (working) esp-box microphone path uses. The T-Deck populates MIC1 and // MIC3, one from each ADC pair, so both pairs' HPFs are enabled. - std::error_code hpf_ec; - es7210_i2c_device_->write_register(0x22, std::vector{0x0a}, hpf_ec); // ADC1/2 HPF1 - es7210_i2c_device_->write_register(0x23, std::vector{0x2a}, hpf_ec); // ADC1/2 HPF2 - es7210_i2c_device_->write_register(0x20, std::vector{0x0a}, hpf_ec); // ADC3/4 HPF2 - es7210_i2c_device_->write_register(0x21, std::vector{0x2a}, hpf_ec); // ADC3/4 HPF1 - if (hpf_ec) { - logger_.warn("Could not enable the ES7210 high-pass filter: {}", hpf_ec.message()); + // write_register clears the error code on success, so a later successful + // write would mask an earlier failure; remember the first failure explicitly. + std::error_code hpf_ec, first_hpf_ec; + auto write_hpf = [&](uint8_t reg, uint8_t val) { + es7210_i2c_device_->write_register(reg, std::vector{val}, hpf_ec); + if (hpf_ec && !first_hpf_ec) { + first_hpf_ec = hpf_ec; + } + }; + write_hpf(0x22, 0x0a); // ADC1/2 HPF1 + write_hpf(0x23, 0x2a); // ADC1/2 HPF2 + write_hpf(0x20, 0x0a); // ADC3/4 HPF2 + write_hpf(0x21, 0x2a); // ADC3/4 HPF1 + if (first_hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", first_hpf_ec.message()); } // apply the stored microphone volume (the driver's init leaves the analog // gain at its 0 dB minimum, which records very quietly) From 1ab2fcc9107204ef38ce117db383fa8474a7ccdc Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 23 Jul 2026 11:30:27 -0500 Subject: [PATCH 12/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../smartpanlee-sc01-plus/example/main/gui.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/components/smartpanlee-sc01-plus/example/main/gui.cpp b/components/smartpanlee-sc01-plus/example/main/gui.cpp index d11dc84ae..e06cbe93b 100644 --- a/components/smartpanlee-sc01-plus/example/main/gui.cpp +++ b/components/smartpanlee-sc01-plus/example/main/gui.cpp @@ -107,11 +107,20 @@ void Gui::init_audio_controls() { void Gui::update_audio_label() { auto &board = espp::SmartPanleeSc01Plus::get(); + static int last_volume = -1; + static bool last_muted = false; + int volume = static_cast(board.volume()); + bool muted = board.is_muted(); + if (volume == last_volume && muted == last_muted) { + return; + } + last_volume = volume; + last_muted = muted; // pass the LVGL symbol as a %s argument rather than concatenating it into the // format-string literal: cppcheck cannot expand the LVGL symbol macros and // flags the literal concatenation as an unknown macro - lv_label_set_text_fmt(audio_label_, "%s %d%%%s", LV_SYMBOL_VOLUME_MAX, - static_cast(board.volume()), board.is_muted() ? " (muted)" : ""); + lv_label_set_text_fmt(audio_label_, "%s %d%%%s", LV_SYMBOL_VOLUME_MAX, volume, + muted ? " (muted)" : ""); } void Gui::init_circle_layer() { From 0d59a1ac1874b86a2dd870584f3a3bdbe7c10165 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 23 Jul 2026 11:30:37 -0500 Subject: [PATCH 13/17] address comment --- components/m5stack-tab5/src/audio.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index 17ec63e1e..501388596 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -226,9 +226,6 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, {.callback = std::bind(&M5StackTab5::microphone_task_callback, this, _1, _2, _3), .task_config = mic_task_config}); - // Enable speaker output - enable_audio(true); - // Start the tasks stepwise so a failure does not leave a half-initialized // state: if the microphone task fails to start, stop the already-running // playback task and report failure rather than leaving it running with @@ -243,6 +240,9 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, return false; } + // Enable speaker output + enable_audio(true); + audio_initialized_ = true; return true; } From e56946ad4da45d67e03f68dd188426cd97f74eb0 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 23 Jul 2026 11:59:46 -0500 Subject: [PATCH 14/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- components/t-deck/src/audio.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/components/t-deck/src/audio.cpp b/components/t-deck/src/audio.cpp index b2dfeed0e..22dace6df 100644 --- a/components/t-deck/src/audio.cpp +++ b/components/t-deck/src/audio.cpp @@ -382,9 +382,16 @@ bool TDeck::initialize_microphone(const microphone_callback_t &callback, uint32_ .task_config = task_config, }); - microphone_initialized_ = true; + if (!microphone_task_->start()) { + i2s_channel_disable(audio_rx_handle); + i2s_del_channel(audio_rx_handle); + audio_rx_handle = nullptr; + microphone_task_.reset(); + return false; + } - return microphone_task_->start(); + microphone_initialized_ = true; + return true; } uint32_t TDeck::microphone_sample_rate() const { return mic_sample_rate_; } From 330f0fd9739370227100b2f66b72de5627fe2d00 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 23 Jul 2026 12:00:30 -0500 Subject: [PATCH 15/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- components/esp-box/src/audio.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/esp-box/src/audio.cpp b/components/esp-box/src/audio.cpp index 5db533bff..48d154e3b 100644 --- a/components/esp-box/src/audio.cpp +++ b/components/esp-box/src/audio.cpp @@ -363,9 +363,14 @@ bool EspBox::initialize_microphone(const microphone_callback_t &callback, .task_config = task_config, }); - microphone_initialized_ = true; + if (!microphone_task_->start()) { + i2s_channel_disable(audio_rx_handle); + microphone_task_.reset(); + return false; + } - return microphone_task_->start(); + microphone_initialized_ = true; + return true; } bool EspBox::microphone_task_callback(std::mutex &m, std::condition_variable &cv, From 4576726b01641815f386fb5f280db4ecbc4e2d5f Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 23 Jul 2026 12:00:41 -0500 Subject: [PATCH 16/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- components/esp32-p4-function-ev-board/src/audio.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/esp32-p4-function-ev-board/src/audio.cpp b/components/esp32-p4-function-ev-board/src/audio.cpp index f015075ff..fa1832704 100644 --- a/components/esp32-p4-function-ev-board/src/audio.cpp +++ b/components/esp32-p4-function-ev-board/src/audio.cpp @@ -213,9 +213,14 @@ bool Esp32P4FunctionEvBoard::initialize_microphone(const microphone_callback_t & .task_config = task_config, }); - microphone_initialized_ = true; + if (!microphone_task_->start()) { + i2s_channel_disable(audio_rx_handle); + microphone_task_.reset(); + return false; + } - return microphone_task_->start(); + microphone_initialized_ = true; + return true; } bool Esp32P4FunctionEvBoard::microphone_task_callback(std::mutex &m, std::condition_variable &cv, From 48b845c01ce5b5213faee14b1a34a23179f120b3 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 23 Jul 2026 13:48:59 -0500 Subject: [PATCH 17/17] minor update to address comment --- components/smartpanlee-sc01-plus/example/main/gui.cpp | 8 +++----- components/smartpanlee-sc01-plus/example/main/gui.hpp | 4 ++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/components/smartpanlee-sc01-plus/example/main/gui.cpp b/components/smartpanlee-sc01-plus/example/main/gui.cpp index e06cbe93b..607b95a28 100644 --- a/components/smartpanlee-sc01-plus/example/main/gui.cpp +++ b/components/smartpanlee-sc01-plus/example/main/gui.cpp @@ -107,15 +107,13 @@ void Gui::init_audio_controls() { void Gui::update_audio_label() { auto &board = espp::SmartPanleeSc01Plus::get(); - static int last_volume = -1; - static bool last_muted = false; int volume = static_cast(board.volume()); bool muted = board.is_muted(); - if (volume == last_volume && muted == last_muted) { + if (volume == last_volume_ && muted == last_muted_) { return; } - last_volume = volume; - last_muted = muted; + last_volume_ = volume; + last_muted_ = muted; // pass the LVGL symbol as a %s argument rather than concatenating it into the // format-string literal: cppcheck cannot expand the LVGL symbol macros and // flags the literal concatenation as an unknown macro diff --git a/components/smartpanlee-sc01-plus/example/main/gui.hpp b/components/smartpanlee-sc01-plus/example/main/gui.hpp index e4113761c..368976e9b 100644 --- a/components/smartpanlee-sc01-plus/example/main/gui.hpp +++ b/components/smartpanlee-sc01-plus/example/main/gui.hpp @@ -138,6 +138,10 @@ class Gui { lv_obj_t *volume_down_button_{nullptr}; lv_obj_t *volume_up_button_{nullptr}; lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_volume_{-1}; + bool last_muted_{false}; lv_obj_t *circle_layer_{nullptr}; audio_button_callback_t play_callback_{nullptr};