diff --git a/components/esp-box/README.md b/components/esp-box/README.md index c8c15ff21..6aa5d7208 100644 --- a/components/esp-box/README.md +++ b/components/esp-box/README.md @@ -8,7 +8,11 @@ and expansion headers. The `espp::EspBox` component provides a singleton hardware abstraction for initializing the touch, display, and audio subsystems, as well as automatically -determining which version of the Box it's running on. +determining which version of the Box it's running on. The audio subsystem runs +full duplex: `play_audio()` streams 16-bit stereo out through the ES8311 codec +and speaker, while `initialize_microphone()` delivers 16-bit stereo recordings +from the dual microphone array (through the ES7210 ADC, with adjustable gain +via `microphone_volume()`) at the speaker's sample rate. ## Example diff --git a/components/esp-box/example/README.md b/components/esp-box/example/README.md index dba524ce9..7bb168823 100644 --- a/components/esp-box/example/README.md +++ b/components/esp-box/example/README.md @@ -9,6 +9,16 @@ state and each time you touch the scren it 1) uses LVGL to draw a circle where you touch, and 2) play a click sound (wav file bundled with the firmware). If you press the home button on the display, it will clear the circles. +The on-screen audio row (bottom-left) exercises the microphones and speaker in +full duplex: the record button records from the dual microphone array (ES7210) +into a PSRAM-preferred buffer, the play button streams the recording back +through the speaker, and the remaining buttons adjust the speaker and +microphone volumes (shown in the label above the row). The measured effective +capture rate is logged when a recording stops. Because the board's +speaker is mono, the recorded stereo (mic 1 / mic 2) is downmixed to both +channels before playback, and the per-channel peak amplitude is logged so +you can tell a silent capture (peak near 0) from a playback issue. + https://github.com/user-attachments/assets/6fc9ce02-fdae-4f36-9ee7-3a6c347ca1c4 https://github.com/esp-cpp/espp/assets/213467/d5379983-9bc2-4d56-a9fc-9e37f54af15e @@ -17,6 +27,24 @@ https://github.com/esp-cpp/espp/assets/213467/d5379983-9bc2-4d56-a9fc-9e37f54af1 ![image](https://github.com/esp-cpp/espp/assets/213467/c7216cfd-330d-4610-baf2-30001c98ff42) ![image](https://github.com/user-attachments/assets/6d9901f1-f4fe-433a-b6d5-c8c82abd14b7) +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_ESP_BOX_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. + +The example also enables `CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK`, which +turns any future task-stack overflow into an immediate, clearly-named panic +instead of silent heap corruption. + ## How to use example ### Hardware Required diff --git a/components/esp-box/example/main/esp_box_example.cpp b/components/esp-box/example/main/esp_box_example.cpp index 4d6a2a099..ec908eef3 100644 --- a/components/esp-box/example/main/esp_box_example.cpp +++ b/components/esp-box/example/main/esp_box_example.cpp @@ -1,9 +1,16 @@ +#include +#include #include #include +#include +#include #include #include #include +#include +#include + #include "esp-box.hpp" #include "gui.hpp" @@ -17,6 +24,21 @@ static std::vector audio_bytes; static bool load_audio(size_t &out_size, size_t &out_sample_rate); static void play_click(espp::EspBox &box); +// Audio recording state (written by the microphone callback, read/controlled +// from the GUI button callbacks and main loop). The recorded data is 16-bit +// interleaved stereo at the current audio sample rate. +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 2; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static std::atomic recording_start_us{0}; +static std::atomic recording_last_us{0}; + extern "C" void app_main(void) { espp::Logger logger({.tag = "ESP BOX Example", .level = espp::Logger::Verbosity::INFO}); logger.info("Starting example!"); @@ -98,8 +120,9 @@ extern "C" void app_main(void) { // directly. static Gui gui({.log_level = espp::Logger::Verbosity::INFO}); static const std::string instructions = - fmt::format("\n\n\n\nTouch the screen!\nPress the home button or the {} button to clear " - "circles.\nPress the {} button to rotate the display.", + fmt::format("Touch the screen to draw!\nPress the home button or the {} button to clear " + "circles.\nPress the {} button to rotate the display.\nThe IMU and Audio " + "tabs show the other subsystems.", LV_SYMBOL_TRASH, LV_SYMBOL_REFRESH); gui.set_label_text(instructions); @@ -119,13 +142,18 @@ extern "C" void app_main(void) { if (touchpad_data.btn_state) { gui.clear_circles(); } - // if there is a touch point, draw a circle and play a click sound - if (touchpad_data.num_touch_points > 0) { + // if there is a touch point on the Draw tab, draw a circle and play a + // click sound (touches on the other tabs go to their widgets) + if (touchpad_data.num_touch_points > 0 && gui.draw_page_active()) { play_click(box); gui.draw_circle(touchpad_data.x, touchpad_data.y, 10); } } }; + // NOTE: this example raises the BSP interrupt-task stack size via + // sdkconfig.defaults (CONFIG_ESP_BOX_INTERRUPT_STACK_SIZE=8192); the + // touch controller is read from that task and its error-logging path + // needs more than the 4 KB BSP default. See the example README. if (!box.initialize_touch(touch_callback)) { logger.error("Failed to initialize touchpad!"); return; @@ -150,6 +178,95 @@ extern "C" void app_main(void) { // set the display brightness to be 75% box.brightness(75.0f); + // Initialize the microphones (the ES7210 shares the I2S bus with the + // speaker in full duplex, so recording runs at the speaker's sample rate) + // and buffer the recorded stereo frames; the recording auto-stops when the + // buffer is full and the main loop notices and updates the GUI + auto mic_callback = [](const uint8_t *data, size_t num_bytes) { + if (!recording) { + return; + } + size_t offset = recording_len; + size_t to_copy = std::min(num_bytes, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = esp_timer_get_time(); + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + recording = false; + } + }; + bool have_mic = box.initialize_microphone(mic_callback); + if (!have_mic) { + gui.set_audio_status("Mic unavailable (see log)"); + } + if (have_mic) { + // start at a modest microphone gain: the BSP default is fairly hot and + // can clip on close / loud sound. Nudge it up with the mic + button if + // recordings are too quiet. + box.microphone_volume(40.0f); + // allocate the recording buffer (16-bit interleaved stereo at the + // current sample rate): prefer PSRAM, fall back to a couple of seconds + // in internal RAM + size_t bytes_per_second = box.audio_sample_rate() * 2 * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + logger.warn("Could not allocate a recording buffer; recording disabled"); + gui.set_audio_status("No recording buffer"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz stereo)", recording_capacity / 1024, + recording_capacity / bytes_per_second, box.audio_sample_rate()); + } + } else { + logger.warn("Could not initialize the microphone!"); + } + + // The record button toggles recording; the play button toggles playback of + // the recording (streamed to the speaker by the main loop) + gui.set_record_callback([&]() { + if (!have_mic || recording_capacity == 0) { + logger.warn("Recording unavailable (no microphone / no buffer)"); + gui.set_audio_status("Mic unavailable (see log)"); + return; + } + if (recording) { + recording = false; // the main loop notices and logs the summary + } else { + playing = false; + gui.set_play_active(false); + recording_len = 0; + recording_start_us = esp_timer_get_time(); + recording_last_us = recording_start_us.load(); + recording = true; + gui.set_record_active(true); + gui.set_audio_status("Recording..."); + } + }); + gui.set_play_callback([&]() { + if (playing) { + playing = false; + gui.set_play_active(false); + gui.set_audio_status("Playback stopped"); + } else if (recording_len > 0) { + recording = false; + playing = true; + gui.set_play_active(true); + gui.set_audio_status("Playing..."); + } else { + logger.info("Nothing recorded yet; press the record button first"); + gui.set_audio_status("Nothing recorded yet"); + } + }); + // make a task to read out the IMU data and update the GUI with it espp::Task imu_task( {.callback = [&](std::mutex &m, std::condition_variable &cv) -> bool { @@ -185,7 +302,7 @@ extern "C" void app_main(void) { gravity_vector.y = -gravity_vector.y; } - std::string text = fmt::format("{}\n\n\n\n\n", instructions); + std::string text; text += fmt::format("Accel: {:02.2f} {:02.2f} {:02.2f}\n", accel.x, accel.y, accel.z); text += fmt::format("Gyro: {:03.2f} {:03.2f} {:03.2f}\n", espp::deg_to_rad(gyro.x), espp::deg_to_rad(gyro.y), espp::deg_to_rad(gyro.z)); @@ -207,7 +324,7 @@ extern "C" void app_main(void) { // update the GUI with the new data; the Gui handles remapping the // vectors for the current display rotation - gui.set_label_text(text); + gui.set_imu_text(text); gui.set_kalman_down(gravity_vector.x, gravity_vector.y); gui.set_madgwick_down(vx, vy); @@ -221,9 +338,67 @@ extern "C" void app_main(void) { }}); imu_task.start(); - // loop forever + // Main loop: stream any active playback to the speaker and notice when a + // recording stops (either button press or the buffer filling up) + size_t play_offset = 0; + bool was_recording = false; while (true) { - std::this_thread::sleep_for(1s); + // feed the active playback in chunks, advancing by however much the + // speaker's stream buffer accepted + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_play_active(false); + gui.set_audio_status("Playback done"); + logger.info("Playback done"); + } else { + play_offset += box.play_audio(recording_buffer + play_offset, + std::min(len - play_offset, 16384)); + } + } else { + play_offset = 0; + } + // notice when the recording stopped (button press or buffer full) + bool now_recording = recording; + if (was_recording && !now_recording) { + gui.set_record_active(false); + gui.set_audio_status(fmt::format("Recorded {:.1f}s ({} plays)", + static_cast(recording_len) / + (box.audio_sample_rate() * 2 * sizeof(int16_t)), + LV_SYMBOL_PLAY)); + // report the measured capture rate: stereo frames recorded over the + // wall clock they took to arrive should match the nominal sample rate + size_t num_frames = recording_len / (2 * sizeof(int16_t)); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_frames / elapsed_s : 0.0f; + // Post-process the recording for playback on the box's MONO speaker: the + // ES7210 records 16-bit interleaved stereo (mic 1 -> left slot, mic 2 -> + // right), but the speaker only plays one I2S slot, so audio captured on + // the other slot would be inaudible. Downmix each L/R frame to the + // average and write it to BOTH slots so the mono speaker always plays + // it. Also report the peak amplitude per channel: a peak near 0 means + // the microphones captured silence (a codec / wiring problem), while a + // healthy peak means capture is fine. + auto *samples = reinterpret_cast(recording_buffer); + int16_t peak_left = 0, peak_right = 0; + for (size_t i = 0; i < num_frames; i++) { + int16_t l = samples[2 * i]; + int16_t r = samples[2 * i + 1]; + peak_left = std::max(peak_left, static_cast(std::abs(l))); + peak_right = std::max(peak_right, static_cast(std::abs(r))); + int16_t mono = static_cast((static_cast(l) + r) / 2); + samples[2 * i] = mono; + samples[2 * i + 1] = mono; + } + logger.info("Recorded {} frames in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal); " + "peak L={} R={} (of 32767 - near 0 means the mics captured silence)", + num_frames, elapsed_s, effective_hz, box.audio_sample_rate(), peak_left, + peak_right); + } + was_recording = now_recording; + std::this_thread::sleep_for(50ms); } //! [esp box example] } @@ -257,13 +432,21 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::EspBox &box) { - // use the box.play_audio() function to play a sound, breaking it into - // audio_buffer_size chunks + // Enqueue the click without blocking the caller (this runs in the touch + // callback). play_audio() enqueues only whole frames and returns how many + // bytes it took, so advance by that count (advancing by the requested size + // would skip samples). Stop as soon as the stream buffer is full rather than + // waiting for it to drain - blocking here would freeze the touch task for the + // whole click. The click comfortably fits in the stream buffer, so it plays + // in full in practice. auto audio_buffer_size = box.audio_buffer_size(); size_t offset = 0; while (offset < audio_bytes.size()) { - size_t bytes_to_play = std::min(audio_buffer_size, audio_bytes.size() - offset); - box.play_audio(audio_bytes.data() + offset, bytes_to_play); - offset += bytes_to_play; + size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); + size_t queued = box.play_audio(audio_bytes.data() + offset, chunk); + offset += queued; + if (queued < chunk) { + break; // stream buffer full for now; do not block the caller + } } } diff --git a/components/esp-box/example/main/gui.cpp b/components/esp-box/example/main/gui.cpp index f2c07468d..8a10ab668 100644 --- a/components/esp-box/example/main/gui.cpp +++ b/components/esp-box/example/main/gui.cpp @@ -1,18 +1,15 @@ +#include #include #include "gui.hpp" void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); - init_label(); - init_gravity_lines(); - init_buttons(); + init_tabview(); + init_draw_tab(); + init_imu_tab(); + init_audio_tab(); init_circle_layer(); - // disable scrolling on the screen (so that it doesn't behave weirdly when - // rotated and drawing with your finger) - lv_obj_set_scrollbar_mode(lv_screen_active(), LV_SCROLLBAR_MODE_OFF); - lv_obj_clear_flag(lv_screen_active(), LV_OBJ_FLAG_SCROLLABLE); } void Gui::deinit_ui() { @@ -20,31 +17,67 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &box = espp::EspBox::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, box.lcd_width(), box.lcd_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(0, 0, 0), 0); +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); + imu_tab_ = lv_tabview_add_tab(tabview_, "IMU"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } -void Gui::init_label() { - label_ = lv_label_create(lv_screen_active()); +void Gui::init_draw_tab() { + // instructions on the left, with the rotate / clear buttons on the right + label_ = lv_label_create(draw_tab_); + lv_label_set_long_mode(label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(label_, lv_pct(70)); lv_label_set_text(label_, ""); lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 0, 0); lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_LEFT, 0); + + rotate_button_ = lv_btn_create(draw_tab_); + lv_obj_set_size(rotate_button_, 50, 50); + lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, 0, 0); + lv_obj_t *rotate_label = lv_label_create(rotate_button_); + lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); + lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); + + clear_button_ = lv_btn_create(draw_tab_); + lv_obj_set_size(clear_button_, 50, 50); + lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 60); + lv_obj_t *clear_label = lv_label_create(clear_button_); + lv_label_set_text(clear_label, LV_SYMBOL_TRASH); + lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); } -void Gui::init_gravity_lines() { - auto &box = espp::EspBox::get(); +void Gui::init_imu_tab() { + imu_label_ = lv_label_create(imu_tab_); + lv_label_set_long_mode(imu_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(imu_label_, lv_pct(100)); + lv_label_set_text(imu_label_, "IMU initializing..."); + lv_obj_align(imu_label_, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_set_style_text_align(imu_label_, LV_TEXT_ALIGN_LEFT, 0); lv_style_init(&kalman_line_style_); lv_style_set_line_width(&kalman_line_style_, 8); lv_style_set_line_color(&kalman_line_style_, lv_palette_main(LV_PALETTE_BLUE)); lv_style_set_line_rounded(&kalman_line_style_, true); - kalman_line_ = lv_line_create(lv_screen_active()); + kalman_line_ = lv_line_create(imu_tab_); kalman_line_points_[0] = {0, 0}; - kalman_line_points_[1] = {box.lcd_width(), box.lcd_height()}; + kalman_line_points_[1] = {0, 0}; lv_line_set_points(kalman_line_, kalman_line_points_, 2); lv_obj_add_style(kalman_line_, &kalman_line_style_, 0); @@ -53,46 +86,92 @@ void Gui::init_gravity_lines() { lv_style_set_line_color(&madgwick_line_style_, lv_palette_main(LV_PALETTE_RED)); lv_style_set_line_rounded(&madgwick_line_style_, true); - madgwick_line_ = lv_line_create(lv_screen_active()); + madgwick_line_ = lv_line_create(imu_tab_); madgwick_line_points_[0] = {0, 0}; - madgwick_line_points_[1] = {box.lcd_width(), box.lcd_height()}; + madgwick_line_points_[1] = {0, 0}; lv_line_set_points(madgwick_line_, madgwick_line_points_, 2); lv_obj_add_style(madgwick_line_, &madgwick_line_style_, 0); } -void Gui::init_buttons() { - // a button in the top left which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(rotate_button_, 50, 50); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_LEFT, 0, 0); - lv_obj_t *rotate_label = lv_label_create(rotate_button_); - lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); - lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); +void Gui::init_audio_tab() { + // a column: status line, volume line, then the buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 10, 0); - // a button in the top right which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(clear_button_, 50, 50); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 0); - lv_obj_t *clear_label = lv_label_create(clear_button_); - lv_label_set_text(clear_label, LV_SYMBOL_TRASH); - lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); + audio_status_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_status_label_, lv_pct(100)); + lv_label_set_text(audio_status_label_, "Idle"); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 10, 0); + lv_obj_set_style_pad_row(row, 10, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&record_button_, LV_SYMBOL_AUDIO}, {&play_button_, LV_SYMBOL_PLAY}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + {&mic_down_button_, LV_SYMBOL_MINUS}, {&mic_up_button_, LV_SYMBOL_PLUS}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 44, 44); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } + // remember the record / play button labels so set_record_active / + // set_play_active can swap their symbols + record_button_label_ = lv_obj_get_child(record_button_, 0); + play_button_label_ = lv_obj_get_child(play_button_, 0); + // color the volume buttons so the speaker and microphone pairs are + // distinguishable from each other + lv_obj_set_style_bg_color(mic_down_button_, lv_palette_main(LV_PALETTE_TEAL), 0); + lv_obj_set_style_bg_color(mic_up_button_, lv_palette_main(LV_PALETTE_TEAL), 0); } -void Gui::init_circle_layer() { +void Gui::update_audio_label() { auto &box = espp::EspBox::get(); + int speaker_volume = static_cast(box.volume()); + int mic_volume = static_cast(box.microphone_volume()); + // this is called every GUI tick; only reformat the label when a value + // actually changes (lv_label_set_text_fmt formats a new string each call) + if (speaker_volume == last_speaker_volume_ && mic_volume == last_mic_volume_) { + return; + } + last_speaker_volume_ = speaker_volume; + last_mic_volume_ = mic_volume; + // Pass the LVGL symbols as %s arguments rather than concatenating them into + // the format-string literal: cppcheck cannot expand the LVGL symbol macros + // and flags the literal concatenation as an unknown macro. + lv_label_set_text_fmt(audio_label_, "Speaker %d%% (%s/%s)\nMic %d%% (teal %s/%s)", speaker_volume, + LV_SYMBOL_VOLUME_MID, LV_SYMBOL_VOLUME_MAX, mic_volume, LV_SYMBOL_MINUS, + LV_SYMBOL_PLUS); +} + +void Gui::init_circle_layer() { + // a transparent, click-through overlay above the tabview which shows the + // touch trail (only populated while the Draw tab is active) circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, box.lcd_width(), box.lcd_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); lv_obj_set_style_bg_opa(circle_layer_, LV_OPA_TRANSP, 0); - lv_obj_set_style_border_width(circle_layer_, 0, 0); - lv_obj_set_style_outline_width(circle_layer_, 0, 0); - lv_obj_set_style_shadow_width(circle_layer_, 0, 0); lv_obj_add_event_cb(circle_layer_, draw_circle_layer, LV_EVENT_DRAW_MAIN, this); lv_obj_move_foreground(circle_layer_); } @@ -101,6 +180,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -116,13 +199,59 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_PRESSED: gui->on_pressed(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_pressed(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &box = espp::EspBox::get(); + if (target == record_button_) { + logger_.info("Record button pressed"); + if (record_callback_) { + record_callback_(); + } + return; + } + if (target == play_button_) { + logger_.info("Play button pressed"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + box.volume(box.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", box.volume()); + update_audio_label(); + return; + } + if (target == mic_down_button_ || target == mic_up_button_) { + float delta = target == mic_down_button_ ? -10.0f : 10.0f; + box.microphone_volume(box.microphone_volume() + delta); + logger_.info("Microphone volume: {:.0f}%", box.microphone_volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button pressed"); next_rotation(); @@ -132,21 +261,53 @@ void Gui::on_pressed(lv_event_t *e) { } } +void Gui::set_record_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(record_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_AUDIO); + lv_obj_set_style_bg_color( + record_button_, active ? lv_palette_main(LV_PALETTE_RED) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_play_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(play_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_PLAY); + lv_obj_set_style_bg_color( + play_button_, active ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_audio_status(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(audio_status_label_, std::string(text).c_str()); +} + void Gui::set_label_text(std::string_view text) { std::lock_guard lock(mutex_); lv_label_set_text(label_, std::string(text).c_str()); } +void Gui::set_imu_text(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(imu_label_, std::string(text).c_str()); +} + +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::next_rotation() { std::lock_guard lock(mutex_); - auto &box = espp::EspBox::get(); clear_circles_impl(); auto rotation = lv_display_get_rotation(lv_display_get_default()); rotation = static_cast((static_cast(rotation) + 1) % 4); lv_display_set_rotation(lv_display_get_default(), rotation); // update the size of the screen-filling objects - lv_obj_set_size(background_, box.rotated_display_width(), box.rotated_display_height()); - lv_obj_set_size(circle_layer_, box.rotated_display_width(), box.rotated_display_height()); + int width = lv_display_get_horizontal_resolution(lv_display_get_default()); + int height = lv_display_get_vertical_resolution(lv_display_get_default()); + lv_obj_set_size(tabview_, width, height); + lv_obj_set_size(circle_layer_, width, height); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_invalidate(circle_layer_); } @@ -162,7 +323,6 @@ void Gui::set_madgwick_down(float vx, float vy) { } void Gui::set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, float vy) { - auto &box = espp::EspBox::get(); // remap the vector according to the current display rotation so that the // line always points toward physical "down" auto rotation = lv_display_get_rotation(lv_display_get_default()); @@ -176,9 +336,16 @@ void Gui::set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, fl std::swap(vx, vy); vy = -vy; } - // draw the line from the center of the screen toward "down" - int x0 = box.rotated_display_width() / 2; - int y0 = box.rotated_display_height() / 2; + // draw the line from the center of the IMU tab's content area toward + // "down" (fall back to the display size if the layout hasn't run yet) + int w = lv_obj_get_content_width(imu_tab_); + int h = lv_obj_get_content_height(imu_tab_); + if (w <= 0 || h <= 0) { + w = lv_display_get_horizontal_resolution(lv_display_get_default()); + h = lv_display_get_vertical_resolution(lv_display_get_default()) - TAB_BAR_HEIGHT; + } + int x0 = w / 2; + int y0 = h / 2; points[0] = {static_cast(x0), static_cast(y0)}; points[1] = {static_cast(x0 + 50 * vx), static_cast(y0 + 50 * vy)}; diff --git a/components/esp-box/example/main/gui.hpp b/components/esp-box/example/main/gui.hpp index b3a2cdffc..88545590f 100644 --- a/components/esp-box/example/main/gui.hpp +++ b/components/esp-box/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -22,15 +23,22 @@ /// dispatched through a single static trampoline (event_callback) into /// member functions, keeping all UI logic inside the class. /// -/// For this example the Gui shows: -/// * A label with instructions and live IMU data -/// * Two lines showing the direction of gravity ("down") as computed by a -/// Kalman filter (blue) and a Madgwick filter (red) -/// * A button (top-left) which rotates the display through 0/90/180/270 -/// * A button (top-right) which clears the circles drawn on the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Draw" tab: instructions, plus the rotate and clear buttons. Touching +/// the screen while this tab is active draws circles (on a transparent +/// overlay) and plays a click. +/// * "IMU" tab: live IMU text and two lines showing the direction of +/// gravity ("down") as computed by a Kalman filter (blue) and a Madgwick +/// filter (red). +/// * "Audio" tab: record / play buttons (wired to the example via +/// callbacks) and speaker / microphone volume buttons (acting directly on +/// the BSP), with a label showing the current volumes. class Gui { public: + /// Callback invoked when the record / play buttons are pressed + using audio_button_callback_t = std::function; + /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -49,10 +57,19 @@ class Gui { deinit_ui(); } - /// Set the text of the main label. Thread-safe. + /// Set the instruction text shown on the Draw tab. Thread-safe. /// @param text The text to display void set_label_text(std::string_view text); + /// Set the live IMU text shown on the IMU tab. Thread-safe. + /// @param text The text to display + void set_imu_text(std::string_view text); + + /// Whether the Draw tab is currently active (used by the example to only + /// draw circles / play clicks for touches on that tab). Thread-safe. + /// @return True if the Draw tab is the active tab + bool draw_page_active(); + /// Draw a circle at the given screen coordinates, replacing the oldest /// circle if the maximum number are already visible. Thread-safe. /// @param x The x coordinate (screen space) @@ -81,8 +98,34 @@ class Gui { /// @param vy The y component of the gravity vector void set_madgwick_down(float vx, float vy); + /// Set the callback invoked when the record button is pressed + /// @param callback The callback to invoke + void set_record_callback(audio_button_callback_t callback) { + record_callback_ = std::move(callback); + } + + /// Set the callback invoked when the play button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + + /// Show whether a recording is in progress (turns the record button red + /// and changes its symbol to stop). Thread-safe. + /// @param active True while recording + void set_record_active(bool active); + + /// Show whether a playback is in progress (turns the play button green and + /// changes its symbol to stop). Thread-safe. + /// @param active True while playing + void set_play_active(bool active); + + /// Set the status line on the Audio tab (e.g. "Recording...", "Mic + /// unavailable"). Thread-safe. + /// @param text The text to display + void set_audio_status(std::string_view text); + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 40; struct Circle { int x{0}; @@ -95,14 +138,19 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); - void init_label(); - void init_gravity_lines(); - void init_buttons(); + void init_tabview(); + void init_draw_tab(); + void init_imu_tab(); + void init_audio_tab(); void init_circle_layer(); + // update the audio volume label from the BSP's current volumes; called + // with the mutex held + void update_audio_label(); + // update the given "down" line from a vector in the unrotated display - // frame, remapping for the current display rotation + // frame, remapping for the current display rotation; called with the + // mutex held void set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, float vy); // the LVGL update task: calls lv_task_handler() under the mutex @@ -112,6 +160,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_pressed(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -122,14 +171,35 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *draw_tab_{nullptr}; + lv_obj_t *imu_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *label_{nullptr}; + lv_obj_t *imu_label_{nullptr}; lv_obj_t *kalman_line_{nullptr}; lv_obj_t *madgwick_line_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *record_button_{nullptr}; + lv_obj_t *record_button_label_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *play_button_label_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *mic_down_button_{nullptr}; + lv_obj_t *mic_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_speaker_volume_{-1}; + int last_mic_volume_{-1}; + lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t record_callback_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + lv_style_t kalman_line_style_; lv_style_t madgwick_line_style_; lv_point_precise_t kalman_line_points_[2]; @@ -140,7 +210,9 @@ class Gui { size_t visible_circle_count_{0}; espp::Task update_task_{{.callback = [this](auto &m, auto &cv) { return update(m, cv); }, - .task_config = {.name = "gui", .stack_size_bytes = 6 * 1024}}}; + // NOTE: rendering the tabview (nested containers + flex layout) uses + // noticeably more stack than a flat UI; 6 KB overflows + .task_config = {.name = "gui", .stack_size_bytes = 12 * 1024}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/esp-box/example/partitions.csv b/components/esp-box/example/partitions.csv new file mode 100644 index 000000000..19d66ab8f --- /dev/null +++ b/components/esp-box/example/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0xa000, 0x6000 +phy_init, data, phy, , 0x1000 +factory, app, factory, , 6M +littlefs, data, littlefs, , 6M diff --git a/components/esp-box/example/sdkconfig.defaults b/components/esp-box/example/sdkconfig.defaults index 152da7c1f..6b5707a51 100644 --- a/components/esp-box/example/sdkconfig.defaults +++ b/components/esp-box/example/sdkconfig.defaults @@ -10,6 +10,12 @@ CONFIG_ESPTOOLPY_FLASHSIZE="16MB" # over twice as fast as DIO CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +# Custom partition table: the tabbed UI + microphone example no longer fits the +# 1 MB default app partition, so use a 6 MB factory app (see partitions.csv). +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_OFFSET=0x9000 +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + # ESP32-specific # CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y @@ -34,3 +40,17 @@ CONFIG_LV_USE_THEME_DEFAULT=y CONFIG_LV_THEME_DEFAULT_DARK=y CONFIG_LV_THEME_DEFAULT_GROW=y CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=30 + +# PSRAM (octal PSRAM on the WROOM-1 R8 / R16V modules); used for the audio +# recording buffer in the example +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y + +# catch task stack overflows immediately (debug watchpoint on the stack end) +# instead of letting them silently corrupt the heap +CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y +# The interrupt task (touch I2C) logs errors through the fmt color-print +# path, which needs more than the 4 KB BSP default; give it 8 KB here so a +# transient touch-bus error cannot overflow its stack. See the README. +CONFIG_ESP_BOX_INTERRUPT_STACK_SIZE=8192 diff --git a/components/esp-box/include/esp-box.hpp b/components/esp-box/include/esp-box.hpp index cee3e549c..38bc9ed26 100644 --- a/components/esp-box/include/esp-box.hpp +++ b/components/esp-box/include/esp-box.hpp @@ -323,13 +323,61 @@ class EspBox : public BaseComponent { float volume() const; /// Play audio - /// \param data The audio data to play - void play_audio(const std::vector &data); + /// \param data The audio data to play (16-bit signed interleaved stereo) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. + size_t play_audio(const std::vector &data); /// Play audio - /// \param data The audio data to play + /// \param data The audio data to play (16-bit signed interleaved stereo) /// \param num_bytes The number of bytes to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. + size_t play_audio(const uint8_t *data, uint32_t num_bytes); + + ///////////////////////////////////////////////////////////////////////////// + // Microphone + ///////////////////////////////////////////////////////////////////////////// + + /// Alias for the microphone callback, called with recorded audio data + using microphone_callback_t = std::function; + + /// Initialize the microphones (the dual microphone array through the + /// ES7210 ADC) and start delivering audio data to the provided callback + /// \param callback The callback to call with recorded audio data: 16-bit + /// signed interleaved stereo (microphone 1 on the left slot, + /// microphone 2 on the right) at audio_sample_rate() + /// \param task_config The configuration for the microphone task + /// \return true if the microphone was successfully initialized, false + /// otherwise + /// \note The sound subsystem must be initialized first (the ES7210 shares + /// the I2S bus with the ES8311 in full duplex, so the microphones + /// record at the speaker's sample rate) + /// \note The callback runs in the microphone task's context, so the task's + /// stack must be large enough for whatever the callback does with + /// the audio data + bool initialize_microphone(const microphone_callback_t &callback, + const espp::Task::BaseConfig &task_config = {.name = "microphone", + .stack_size_bytes = 4096, + .priority = 10, + .core_id = 1}); + + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100), mapped onto the + /// ES7210 analog microphone gain range (0 dB - +37.5 dB) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100) + float microphone_volume() const; ///////////////////////////////////////////////////////////////////////////// // IMU @@ -358,6 +406,7 @@ class EspBox : public BaseComponent { bool update_touch(); void update_volume_output(); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); void lcd_wait_lines(); // box 3: @@ -536,6 +585,15 @@ class EspBox : public BaseComponent { StreamBufferHandle_t audio_tx_stream; i2s_std_config_t audio_std_cfg; + // microphone + std::shared_ptr> es7210_i2c_device_; + std::atomic microphone_initialized_{false}; + microphone_callback_t microphone_callback_{nullptr}; + std::unique_ptr microphone_task_{nullptr}; + std::vector audio_rx_buffer; + // microphone volume (percent), mapped onto the ES7210 analog gain range + std::atomic mic_volume_{70.0f}; + // IMU std::shared_ptr> imu_i2c_device_; std::shared_ptr imu_; diff --git a/components/esp-box/src/audio.cpp b/components/esp-box/src/audio.cpp index 732f3874c..48d154e3b 100644 --- a/components/esp-box/src/audio.cpp +++ b/components/esp-box/src/audio.cpp @@ -1,3 +1,6 @@ +#include +#include + #include "esp-box.hpp" using namespace espp; @@ -6,6 +9,40 @@ using namespace espp; // Audio Functions // //////////////////////// +// Map a 0-100% microphone volume onto the ES7210's analog gain steps +// (GAIN_0DB .. GAIN_37_5DB) +static es7210_gain_value_t microphone_gain_from_volume(float volume) { + int step = static_cast(std::lround(volume / 100.0f * static_cast(GAIN_37_5DB))); + step = std::clamp(step, static_cast(GAIN_0DB), static_cast(GAIN_37_5DB)); + return static_cast(step); +} + +// Map a PCM sample rate onto the codec's audio_hal sample-rate enum. The ES7210 +// coefficient table is selected by this enum, so it must match the rate the I2S +// clocks actually generate; an unsupported rate falls back to 48 kHz. +static audio_hal_iface_samples_t audio_hal_samples_from_rate(uint32_t rate) { + switch (rate) { + case 8000: + return AUDIO_HAL_08K_SAMPLES; + case 11025: + return AUDIO_HAL_11K_SAMPLES; + case 16000: + return AUDIO_HAL_16K_SAMPLES; + case 22050: + return AUDIO_HAL_22K_SAMPLES; + case 24000: + return AUDIO_HAL_24K_SAMPLES; + case 32000: + return AUDIO_HAL_32K_SAMPLES; + case 44100: + return AUDIO_HAL_44K_SAMPLES; + case 48000: + return AUDIO_HAL_48K_SAMPLES; + default: + return AUDIO_HAL_48K_SAMPLES; + } +} + bool EspBox::initialize_codec() { logger_.info("initializing codec"); @@ -122,9 +159,9 @@ void EspBox::enable_sound(bool enable) { gpio_set_level(sound_power_pin, enable) bool EspBox::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { // Queue the next I2S out frame to write - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); uint8_t *buffer = &audio_tx_buffer[0]; memset(buffer, 0, buffer_size); @@ -156,7 +193,7 @@ void EspBox::mute(bool mute) { bool EspBox::is_muted() const { return mute_; } void EspBox::volume(float volume) { - volume_ = volume; + volume_ = std::clamp(volume, 0.0f, 100.0f); update_volume_output(); } @@ -183,9 +220,184 @@ void EspBox::audio_sample_rate(uint32_t sample_rate) { i2s_channel_enable(audio_tx_handle); } -void EspBox::play_audio(const std::vector &data) { play_audio(data.data(), data.size()); } +size_t EspBox::play_audio(const std::vector &data) { + return play_audio(data.data(), data.size()); +} -void EspBox::play_audio(const uint8_t *data, uint32_t num_bytes) { - // don't block here - xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); +size_t EspBox::play_audio(const uint8_t *data, uint32_t num_bytes) { + // guard against being called before initialize_sound() (audio_tx_stream is + // not valid until then) and against empty input + if (!sound_initialized_ || !data || num_bytes == 0) { + return 0; + } + // Enqueue only whole 16-bit stereo frames (4 bytes) that actually fit: + // xStreamBufferSend can accept fewer bytes than requested when the buffer is + // nearly full, and a partial (non-frame-aligned) send would strand 1-3 bytes + // and corrupt the L/R framing of subsequent appends. Cap the request to the + // free space rounded down to a whole frame so the send is always + // frame-aligned. This runs in task context, so the non-ISR send with a 0 + // timeout never blocks. The number of bytes actually queued is returned so + // callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 4; + if (sendable == 0) { + return 0; + } + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); } + +////////////////////////// +// Microphone Functions // +////////////////////////// + +bool EspBox::initialize_microphone(const microphone_callback_t &callback, + const espp::Task::BaseConfig &task_config) { + logger_.info("Initializing microphone"); + if (microphone_initialized_) { + logger_.warn("Microphone already initialized, not initializing again!"); + return false; + } + if (!sound_initialized_) { + logger_.error("The sound subsystem must be initialized first: the ES7210 shares the I2S bus " + "with the ES8311 in full duplex"); + return false; + } + if (!callback) { + logger_.error("A callback is required to receive the recorded audio data"); + return false; + } + microphone_callback_ = callback; + + // The RX channel was allocated alongside the TX channel in + // initialize_i2s(); in full-duplex mode it shares the TX BCLK/WS, so + // initialize it with the same standard-mode (16-bit stereo) config + ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_rx_handle, &audio_std_cfg)); + audio_rx_buffer.resize(calc_audio_buffer_size(audio_sample_rate())); + ESP_ERROR_CHECK(i2s_channel_enable(audio_rx_handle)); + + // Configure the ES7210 ADC (microphone 1 -> left slot, microphone 2 -> + // right slot) now that the I2S clocks are running. Probe for the chip + // first: the address depends on how its AD pins are strapped, so accept + // either 0x40 or 0x41 and report clearly if neither responds. + uint8_t es7210_address = 0; + for (uint8_t address : {0x40, 0x41}) { + if (internal_i2c_.probe_device(address)) { + es7210_address = address; + break; + } + } + if (es7210_address == 0) { + logger_.error("No ES7210 found on the internal I2C bus (probed 0x40 and 0x41); " + "cannot record from the microphones"); + i2s_channel_disable(audio_rx_handle); + return false; + } + logger_.info("Found ES7210 at {:#04x}", es7210_address); + std::error_code ec; + es7210_i2c_device_ = internal_i2c_.add_device( + { + .device_address = es7210_address, + .timeout_ms = static_cast(internal_i2c_.config().timeout_ms), + .scl_speed_hz = internal_i2c_.config().clk_speed, + .log_level = espp::Logger::Verbosity::WARN, + }, + ec); + if (!es7210_i2c_device_) { + logger_.error("Could not initialize ES7210 I2C device: {}", ec.message()); + i2s_channel_disable(audio_rx_handle); + return false; + } + set_es7210_write(espp::make_i2c_addressed_write(es7210_i2c_device_)); + set_es7210_read(espp::make_i2c_addressed_read_register(es7210_i2c_device_)); + + // only the two microphones are wired on the box (the other ES7210 inputs + // are unused), and they map onto the two I2S slots + es7210_mic_select(static_cast(ES7210_INPUT_MIC1 | ES7210_INPUT_MIC2)); + + audio_hal_codec_config_t es7210_cfg{}; + es7210_cfg.codec_mode = AUDIO_HAL_CODEC_MODE_ENCODE; + es7210_cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; + es7210_cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; + es7210_cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; + // configure the ES7210 for the rate the I2S clocks are actually running at + // (rather than assuming 48 kHz) so its coefficient table matches + es7210_cfg.i2s_iface.samples = audio_hal_samples_from_rate(audio_sample_rate()); + if (es7210_adc_init(&es7210_cfg) != ESP_OK) { + logger_.error("Could not initialize the ES7210 codec"); + i2s_channel_disable(audio_rx_handle); + return false; + } + if (es7210_adc_config_i2s(AUDIO_HAL_CODEC_MODE_ENCODE, &es7210_cfg.i2s_iface) != ESP_OK) { + logger_.error("ES7210 I2S config failed"); + } + // Enable the ES7210 digital high-pass filter on the ADC channels to strip + // the microphone DC offset. The espp es7210 driver leaves these registers + // at their reset value (HPF off), so the DC offset is amplified by the + // analog gain and rails the ADC to full scale - recordings come out as a + // near-constant DC level that the AC-coupled speaker plays as a click then + // silence. These are the driver's documented "quick setup" HPF values. + // write_register clears the error code on success, so a later successful + // write would mask an earlier failure; remember the first failure explicitly. + std::error_code hpf_ec, first_hpf_ec; + auto write_hpf = [&](uint8_t reg, uint8_t val) { + es7210_i2c_device_->write_register(reg, std::vector{val}, hpf_ec); + if (hpf_ec && !first_hpf_ec) { + first_hpf_ec = hpf_ec; + } + }; + write_hpf(0x22, 0x0a); // ADC1/2 HPF1 + write_hpf(0x23, 0x2a); // ADC1/2 HPF2 + write_hpf(0x20, 0x0a); // ADC3/4 HPF2 + write_hpf(0x21, 0x2a); // ADC3/4 HPF1 + if (first_hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", first_hpf_ec.message()); + } + // apply the stored microphone volume (the driver's init leaves the analog + // gain at its 0 dB minimum, which records very quietly) + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + es7210_adc_ctrl_state(AUDIO_HAL_CODEC_MODE_ENCODE, AUDIO_HAL_CTRL_START); + + using namespace std::placeholders; + microphone_task_ = espp::Task::make_unique({ + .callback = std::bind(&EspBox::microphone_task_callback, this, _1, _2, _3), + .task_config = task_config, + }); + + if (!microphone_task_->start()) { + i2s_channel_disable(audio_rx_handle); + microphone_task_.reset(); + return false; + } + + microphone_initialized_ = true; + return true; +} + +bool EspBox::microphone_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. + auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), + &bytes_read, pdMS_TO_TICKS(100)); + if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { + microphone_callback_(audio_rx_buffer.data(), bytes_read); + } + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running +} + +void EspBox::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (microphone_initialized_) { + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + } +} + +float EspBox::microphone_volume() const { return mic_volume_; } diff --git a/components/esp32-p4-function-ev-board/README.md b/components/esp32-p4-function-ev-board/README.md index 55f30a1d6..8459f61ce 100644 --- a/components/esp32-p4-function-ev-board/README.md +++ b/components/esp32-p4-function-ev-board/README.md @@ -51,7 +51,11 @@ initializes and exposes the board's peripherals: - **GT911 capacitive multi-touch** — polled by default (the touch INT pin is not routed to the ESP32-P4 on this board), or **interrupt-driven** if you wire the INT pin from the LCD expansion header to a GPIO (see [Touch mode](#touch-mode-polling-vs-interrupt)). -- **ES8311 audio codec** (+ NS4150B speaker amplifier) over I2S for playback. +- **ES8311 audio codec** (+ NS4150B speaker amplifier) over I2S, full duplex: + 16-bit mono playback via `play_audio()` and recording from the onboard + analog microphone via `initialize_microphone()` (16-bit mono at the + speaker's sample rate, with adjustable analog gain via + `microphone_volume()`). - **10/100 Ethernet** (EMAC + IP101 RMII PHY) with DHCP. - **microSD card** (4-bit SDMMC, powered via the on-chip LDO). - **MIPI-CSI camera** (SC2336/OV5647) — pins/SCCB wired, capture pipeline is a diff --git a/components/esp32-p4-function-ev-board/example/README.md b/components/esp32-p4-function-ev-board/example/README.md index 384e2a24d..a0016e4da 100644 --- a/components/esp32-p4-function-ev-board/example/README.md +++ b/components/esp32-p4-function-ev-board/example/README.md @@ -15,7 +15,11 @@ It: card size, Ethernet IP, **RTPS publisher** status, and **system** info (free internal/PSRAM heap and uptime), - mounts the microSD card (if inserted), -- initializes the ES8311 audio codec (and loads an embedded `click.wav`), +- initializes the ES8311 audio codec (and loads an embedded `click.wav`); + the on-screen audio row (bottom-left) records from the onboard microphone + into a PSRAM-preferred buffer, plays the recording back through the + speaker, and adjusts the speaker / microphone volumes (the measured + effective capture rate is logged when a recording stops), - brings up Ethernet (IP101) with DHCP and, once it has an IP, starts an **RTPS participant** that publishes a counter on `espp/test/counter`, and - wires the BOOT button. @@ -25,6 +29,22 @@ It: > ESP32-P4 main board (`RST_LCD` → J1 GPIO27, `PWM` → J1 GPIO26) or the screen > stays black. See the component README. +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_ESP_P4_EV_BOARD_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. +- **Polled touch task stack (`CONFIG_ESP_P4_EV_BOARD_TOUCH_TASK_STACK_SIZE` = 8192).** Same reason: + this board polls the GT911 over I2C from its own task. + ## Configuration Use `idf.py menuconfig` → *ESP32-P4 Function EV Board Configuration* to select diff --git a/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp b/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp index daf674264..1b50f2c36 100644 --- a/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp +++ b/components/esp32-p4-function-ev-board/example/main/esp32_p4_function_ev_board_example.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -39,6 +40,21 @@ static std::vector audio_bytes; static bool load_audio(size_t &out_size, size_t &out_sample_rate); +// Audio recording state (written by the microphone callback, read/controlled +// from the GUI button callbacks and main loop). The recorded data is 16-bit +// mono at the audio sample rate. +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 2; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static std::atomic recording_start_us{0}; +static std::atomic recording_last_us{0}; + // Ping a target host a few times and log the result (uses the espp Ping helper). static void ping_target(espp::Logger &logger, const char *name, const std::string &ip) { if (ip.empty() || ip == "0.0.0.0") { @@ -187,6 +203,11 @@ extern "C" void app_main(void) { // touch-down or once the point has moved at least one radius, so a stationary // touch draws a single circle and a drag leaves a spaced trail. static constexpr int kCircleRadius = 10; + // NOTE: this example raises the BSP interrupt- and touch-task stack sizes + // via sdkconfig.defaults (CONFIG_ESP_P4_EV_BOARD_INTERRUPT_STACK_SIZE / + // _TOUCH_TASK_STACK_SIZE = 8192); the touch controller is read (polled) + // from those tasks and their error-logging path needs more than the 4 KB + // BSP default. See the example README. board.initialize_touch([&](const auto &data) { auto td = board.touchpad_convert(data); static Board::TouchpadData prev_td = {}; @@ -198,7 +219,7 @@ extern "C" void app_main(void) { if (new_touch && !audio_bytes.empty()) { board.play_audio(audio_bytes); // non-blocking, touch-down edge only } - if (new_touch) { + if (new_touch && gui.draw_page_active()) { gui.draw_circle(td.x, td.y, kCircleRadius); } } @@ -229,8 +250,89 @@ extern "C" void app_main(void) { if (have_audio) { logger.info("Loaded {} bytes of click audio @ {} Hz", wav_size, wav_sample_rate); } + + // Microphone: the ES8311 is full duplex, so the onboard microphone + // records at the speaker's sample rate. Buffer the recorded mono samples + // and auto-stop when the buffer is full (the main loop notices and + // updates the GUI). + auto mic_callback = [](const uint8_t *data, size_t num_bytes) { + if (!recording) { + return; + } + size_t offset = recording_len; + size_t to_copy = std::min(num_bytes, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = esp_timer_get_time(); + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + recording = false; + } + }; + if (board.initialize_microphone(mic_callback)) { + // allocate the recording buffer (16-bit mono at the current sample + // rate): prefer PSRAM, fall back to a couple of seconds in internal RAM + size_t bytes_per_second = board.audio_sample_rate() * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + logger.warn("Could not allocate a recording buffer; recording disabled"); + gui.set_audio_status("No recording buffer"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz mono)", recording_capacity / 1024, + recording_capacity / bytes_per_second, board.audio_sample_rate()); + } + } else { + logger.warn("Could not initialize the microphone!"); + gui.set_audio_status("Mic unavailable (see log)"); + } } + // The record button toggles recording; the play button toggles playback of + // the recording (streamed to the speaker by the main loop) + gui.set_record_callback([&]() { + if (recording_capacity == 0) { + logger.warn("Recording unavailable (no microphone / no buffer)"); + gui.set_audio_status("Mic unavailable (see log)"); + return; + } + if (recording) { + recording = false; // the main loop notices and logs the summary + } else { + playing = false; + gui.set_play_active(false); + recording_len = 0; + recording_start_us = esp_timer_get_time(); + recording_last_us = recording_start_us.load(); + recording = true; + gui.set_record_active(true); + gui.set_audio_status("Recording..."); + } + }); + gui.set_play_callback([&]() { + if (playing) { + playing = false; + gui.set_play_active(false); + gui.set_audio_status("Playback stopped"); + } else if (recording_len > 0) { + recording = false; + playing = true; + gui.set_play_active(true); + gui.set_audio_status("Playing..."); + } else { + logger.info("Nothing recorded yet; press the record button first"); + gui.set_audio_status("Nothing recorded yet"); + } + }); + // Ethernet (IP101) — DHCP; the callback fires once an IP is acquired static std::atomic have_ip{false}; static std::string ip_str{"(no link)"}; @@ -380,6 +482,43 @@ extern "C" void app_main(void) { rtps_value = value; rtps_has_peers = published; + // Stream any active playback to the speaker in chunks, advancing by + // however much the stream buffer accepted + static size_t play_offset = 0; + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_play_active(false); + gui.set_audio_status("Playback done"); + logger.info("Playback done"); + } else { + play_offset += board.play_audio(recording_buffer + play_offset, + std::min(len - play_offset, 16384)); + } + } else { + play_offset = 0; + } + // Notice when the recording stopped (button press or buffer full) + static bool was_recording = false; + bool now_recording = recording; + if (was_recording && !now_recording) { + gui.set_record_active(false); + gui.set_audio_status(fmt::format("Recorded {:.1f}s ({} plays)", + static_cast(recording_len) / + (board.audio_sample_rate() * sizeof(int16_t)), + LV_SYMBOL_PLAY)); + // report the measured capture rate: samples recorded over the wall + // clock they took to arrive should match the nominal sample rate + size_t num_samples = recording_len / sizeof(int16_t); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_samples / elapsed_s : 0.0f; + logger.info("Recorded {} samples in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal)", + num_samples, elapsed_s, effective_hz, board.audio_sample_rate()); + } + was_recording = now_recording; + std::this_thread::sleep_for(loop_tick); } } diff --git a/components/esp32-p4-function-ev-board/example/main/gui.cpp b/components/esp32-p4-function-ev-board/example/main/gui.cpp index 2bf3c3b6f..36d6fd751 100644 --- a/components/esp32-p4-function-ev-board/example/main/gui.cpp +++ b/components/esp32-p4-function-ev-board/example/main/gui.cpp @@ -1,17 +1,35 @@ +#include +#include + #include "gui.hpp" using Board = espp::Esp32P4FunctionEvBoard; void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); + init_tabview(); init_labels(); init_buttons(); + init_audio_controls(); init_circle_layer(); - // disable scrolling on the screen (so that it doesn't behave weirdly when - // rotated and drawing with your finger) - lv_obj_set_scrollbar_mode(lv_screen_active(), LV_SCROLLBAR_MODE_OFF); - lv_obj_clear_flag(lv_screen_active(), LV_OBJ_FLAG_SCROLLABLE); +} + +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + status_tab_ = lv_tabview_add_tab(tabview_, "Status"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } void Gui::deinit_ui() { @@ -19,53 +37,137 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &board = Board::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, board.display_width(), board.display_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(0, 0, 0), 0); -} - void Gui::init_labels() { - // a title label at the top of the screen - title_label_ = lv_label_create(lv_screen_active()); + // a title label at the top of the Status tab + title_label_ = lv_label_create(status_tab_); + lv_label_set_long_mode(title_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(title_label_, lv_pct(100)); lv_label_set_text(title_label_, "ESP32-P4 Function EV Board - touch to draw!"); - lv_obj_align(title_label_, LV_ALIGN_TOP_MID, 0, 8); + lv_obj_align(title_label_, LV_ALIGN_TOP_MID, 0, 0); // a status label showing the live subsystem state, updated via // set_status_text() - status_label_ = lv_label_create(lv_screen_active()); + status_label_ = lv_label_create(status_tab_); + lv_label_set_long_mode(status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(status_label_, lv_pct(100)); lv_label_set_text(status_label_, ""); lv_obj_set_style_text_align(status_label_, LV_TEXT_ALIGN_LEFT, 0); - lv_obj_align(status_label_, LV_ALIGN_TOP_LEFT, 8, 40); + lv_obj_align(status_label_, LV_ALIGN_TOP_LEFT, 0, 32); } void Gui::init_buttons() { - // a button in the top right which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); + // a button in the top right of the Status tab which rotates the display + // through 0/90/180/270 degrees + rotate_button_ = lv_btn_create(status_tab_); lv_obj_set_size(rotate_button_, 50, 50); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, -8, 8); + lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, 0, 0); lv_obj_t *rotate_label = lv_label_create(rotate_button_); lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_CLICKED, this); // a button next to it which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); + clear_button_ = lv_btn_create(status_tab_); lv_obj_set_size(clear_button_, 50, 50); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, -66, 8); + lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, -58, 0); lv_obj_t *clear_label = lv_label_create(clear_button_); lv_label_set_text(clear_label, LV_SYMBOL_TRASH); lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_CLICKED, this); } +void Gui::init_audio_controls() { + // the Audio tab: a column with a status line, the volume line, then the + // buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 12, 0); + + audio_status_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_status_label_, lv_pct(100)); + lv_label_set_text(audio_status_label_, "Idle"); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 12, 0); + lv_obj_set_style_pad_row(row, 12, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&record_button_, LV_SYMBOL_AUDIO}, {&play_button_, LV_SYMBOL_PLAY}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + {&mic_down_button_, LV_SYMBOL_MINUS}, {&mic_up_button_, LV_SYMBOL_PLUS}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 50, 50); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_CLICKED, this); + } + // remember the record / play button labels so set_record_active / + // set_play_active can swap their symbols + record_button_label_ = lv_obj_get_child(record_button_, 0); + play_button_label_ = lv_obj_get_child(play_button_, 0); + // color the volume buttons so the speaker and microphone pairs are + // distinguishable from each other + lv_obj_set_style_bg_color(mic_down_button_, lv_palette_main(LV_PALETTE_TEAL), 0); + lv_obj_set_style_bg_color(mic_up_button_, lv_palette_main(LV_PALETTE_TEAL), 0); +} + +void Gui::update_audio_label() { + auto &board = espp::Esp32P4FunctionEvBoard::get(); + int speaker_volume = static_cast(board.volume()); + int mic_volume = static_cast(board.microphone_volume()); + // this is called every GUI tick; only reformat the label when a value + // actually changes (lv_label_set_text_fmt formats a new string each call) + if (speaker_volume == last_speaker_volume_ && mic_volume == last_mic_volume_) { + return; + } + last_speaker_volume_ = speaker_volume; + last_mic_volume_ = mic_volume; + // Pass the LVGL symbols as %s arguments rather than concatenating them into + // the format-string literal: cppcheck cannot expand the LVGL symbol macros + // and flags the literal concatenation as an unknown macro. + lv_label_set_text_fmt(audio_label_, "Speaker %d%% (%s/%s)\nMic %d%% (teal %s/%s)", speaker_volume, + LV_SYMBOL_VOLUME_MID, LV_SYMBOL_VOLUME_MAX, mic_volume, LV_SYMBOL_MINUS, + LV_SYMBOL_PLUS); +} + +void Gui::set_record_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(record_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_AUDIO); + lv_obj_set_style_bg_color( + record_button_, active ? lv_palette_main(LV_PALETTE_RED) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_play_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(play_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_PLAY); + lv_obj_set_style_bg_color( + play_button_, active ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + void Gui::init_circle_layer() { - auto &board = Board::get(); + // a transparent, click-through overlay above the tabview which shows the + // touch trail (only populated while the Status tab is active) circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, board.display_width(), board.display_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); @@ -81,6 +183,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -96,13 +202,59 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_CLICKED: gui->on_clicked(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_clicked(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &board = espp::Esp32P4FunctionEvBoard::get(); + if (target == record_button_) { + logger_.info("Record button clicked"); + if (record_callback_) { + record_callback_(); + } + return; + } + if (target == play_button_) { + logger_.info("Play button clicked"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + board.volume(board.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", board.volume()); + update_audio_label(); + return; + } + if (target == mic_down_button_ || target == mic_up_button_) { + float delta = target == mic_down_button_ ? -10.0f : 10.0f; + board.microphone_volume(board.microphone_volume() + delta); + logger_.info("Microphone volume: {:.0f}%", board.microphone_volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button clicked"); next_rotation(); @@ -117,6 +269,16 @@ void Gui::set_status_text(std::string_view text) { lv_label_set_text(status_label_, std::string(text).c_str()); } +void Gui::set_audio_status(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(audio_status_label_, std::string(text).c_str()); +} + +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::next_rotation() { std::lock_guard lock(mutex_); auto &board = Board::get(); @@ -125,14 +287,11 @@ void Gui::next_rotation() { rotation = static_cast((static_cast(rotation) + 1) % 4); lv_display_set_rotation(lv_display_get_default(), rotation); // update the size of the screen-filling objects - lv_obj_set_size(background_, board.rotated_display_width(), board.rotated_display_height()); + lv_obj_set_size(tabview_, board.rotated_display_width(), board.rotated_display_height()); lv_obj_set_size(circle_layer_, board.rotated_display_width(), board.rotated_display_height()); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_move_foreground(circle_layer_); lv_obj_invalidate(circle_layer_); - // re-align the labels to the (now reoriented) screen edges - lv_obj_align(title_label_, LV_ALIGN_TOP_MID, 0, 8); - lv_obj_align(status_label_, LV_ALIGN_TOP_LEFT, 8, 40); } void Gui::draw_circle(int x, int y, int radius) { diff --git a/components/esp32-p4-function-ev-board/example/main/gui.hpp b/components/esp32-p4-function-ev-board/example/main/gui.hpp index 8f6836b6f..93501c259 100644 --- a/components/esp32-p4-function-ev-board/example/main/gui.hpp +++ b/components/esp32-p4-function-ev-board/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -26,11 +27,19 @@ /// * A title label at the top of the screen /// * A status label showing live subsystem state (panel, touch, SD, Ethernet, /// RTPS, and system memory/uptime), updated via set_status_text() -/// * A button (top-right) which rotates the display through 0/90/180/270 -/// * A button (next to it) which clears the circles drawn on the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Status" tab: title + live subsystem state (panel, touch, SD, +/// Ethernet, RTPS, memory/uptime) and the rotate / clear buttons. +/// Touching the screen while this tab is active draws circles (on a +/// transparent overlay). +/// * "Audio" tab: record / play buttons (wired to the example via +/// callbacks) and speaker / microphone volume buttons (acting directly on +/// the BSP), with labels showing the audio state and current volumes. class Gui { public: + /// Callback invoked when the record / play buttons are pressed + using audio_button_callback_t = std::function; /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -67,8 +76,39 @@ class Gui { /// re-aligning the UI to match. Thread-safe. void next_rotation(); + /// Whether the Status tab is currently active (used by the example to + /// only draw circles for touches on that tab). Thread-safe. + /// @return True if the Status tab is the active tab + bool draw_page_active(); + + /// Set the status line on the Audio tab (e.g. "Recording...", "Mic + /// unavailable"). Thread-safe. + /// @param text The text to display + void set_audio_status(std::string_view text); + + /// Set the callback invoked when the record button is pressed + /// @param callback The callback to invoke + void set_record_callback(audio_button_callback_t callback) { + record_callback_ = std::move(callback); + } + + /// Set the callback invoked when the play button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + + /// Show whether a recording is in progress (turns the record button red + /// and changes its symbol to stop). Thread-safe. + /// @param active True while recording + void set_record_active(bool active); + + /// Show whether a playback is in progress (turns the play button green and + /// changes its symbol to stop). Thread-safe. + /// @param active True while playing + void set_play_active(bool active); + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 50; struct Circle { int x{0}; @@ -81,11 +121,16 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); + void init_tabview(); void init_labels(); void init_buttons(); + void init_audio_controls(); void init_circle_layer(); + // update the audio volume label from the BSP's current volumes; called + // with the mutex held + void update_audio_label(); + // the LVGL update task: calls lv_task_handler() under the mutex bool update(std::mutex &m, std::condition_variable &cv); @@ -93,6 +138,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_clicked(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -103,19 +149,40 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *status_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *title_label_{nullptr}; lv_obj_t *status_label_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *record_button_{nullptr}; + lv_obj_t *record_button_label_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *play_button_label_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *mic_down_button_{nullptr}; + lv_obj_t *mic_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_speaker_volume_{-1}; + int last_mic_volume_{-1}; + lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t record_callback_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + std::array circles_; size_t next_circle_index_{0}; size_t visible_circle_count_{0}; espp::Task update_task_{{.callback = [this](auto &m, auto &cv) { return update(m, cv); }, - .task_config = {.name = "gui", .stack_size_bytes = 6 * 1024}}}; + // NOTE: rendering the tabview (nested containers + flex layout) uses + // noticeably more stack than a flat UI; 6 KB overflows + .task_config = {.name = "gui", .stack_size_bytes = 12 * 1024}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/esp32-p4-function-ev-board/example/sdkconfig.defaults b/components/esp32-p4-function-ev-board/example/sdkconfig.defaults index 6d3f1dcf8..d18e81de3 100644 --- a/components/esp32-p4-function-ev-board/example/sdkconfig.defaults +++ b/components/esp32-p4-function-ev-board/example/sdkconfig.defaults @@ -51,6 +51,10 @@ CONFIG_LV_THEME_DEFAULT_GROW=y CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=30 # ESP32-P4 Function EV Board BSP configuration -CONFIG_ESP_P4_EV_BOARD_INTERRUPT_STACK_SIZE=4096 -CONFIG_ESP_P4_EV_BOARD_TOUCH_TASK_STACK_SIZE=4096 +# Raised from the 4 KB BSP defaults: the interrupt task and the polled +# touch task run touch-controller I2C and, on a bus error, log through +# espp's fmt color-print path, which needs more than 4 KB of stack. See +# the README. +CONFIG_ESP_P4_EV_BOARD_INTERRUPT_STACK_SIZE=8192 +CONFIG_ESP_P4_EV_BOARD_TOUCH_TASK_STACK_SIZE=8192 CONFIG_ESP_P4_EV_BOARD_AUDIO_TASK_STACK_SIZE=8192 diff --git a/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp b/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp index 207b498c2..e6383ed98 100644 --- a/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp +++ b/components/esp32-p4-function-ev-board/include/esp32-p4-function-ev-board.hpp @@ -270,13 +270,61 @@ class Esp32P4FunctionEvBoard : public BaseComponent { size_t audio_buffer_size() const; /// Play audio data - /// \param data The audio data to play + /// \param data The audio data to play (16-bit signed mono samples) /// \param num_bytes The number of bytes to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. + size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Play audio data - /// \param data The audio data to play - void play_audio(std::span data); + /// \param data The audio data to play (16-bit signed mono samples) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. + size_t play_audio(std::span data); + + ///////////////////////////////////////////////////////////////////////////// + // Microphone + ///////////////////////////////////////////////////////////////////////////// + + /// Alias for the microphone callback, called with recorded audio data + using microphone_callback_t = std::function; + + /// Initialize the microphone (the onboard analog microphone through the + /// ES8311 codec's ADC) and start delivering audio data to the provided + /// callback + /// \param callback The callback to call with recorded audio data (16-bit + /// signed mono samples at audio_sample_rate()) + /// \param task_config The configuration for the microphone task + /// \return true if the microphone was successfully initialized, false + /// otherwise + /// \note The audio subsystem must be initialized first (the ES8311 is a + /// full-duplex codec on a single I2S bus, so the microphone records + /// at the speaker's sample rate) + /// \note The callback runs in the microphone task's context, so the task's + /// stack must be large enough for whatever the callback does with + /// the audio data + bool initialize_microphone(const microphone_callback_t &callback, + const espp::Task::BaseConfig &task_config = {.name = "microphone", + .stack_size_bytes = 4096, + .priority = 10, + .core_id = 1}); + + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100), mapped onto the + /// ES8311 analog microphone gain range (0 dB - +42 dB) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100) + float microphone_volume() const; ///////////////////////////////////////////////////////////////////////////// // uSD Card (4-bit SDMMC) @@ -360,6 +408,7 @@ class Esp32P4FunctionEvBoard : public BaseComponent { bool update_touch(); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); ///////////////////////////////////////////////////////////////////////////// // Display geometry / per-panel parameters @@ -539,6 +588,15 @@ class Esp32P4FunctionEvBoard : public BaseComponent { StreamBufferHandle_t audio_tx_stream{nullptr}; std::atomic has_sound{false}; + // Microphone (ES8311 ADC, full duplex with the speaker) + std::atomic microphone_initialized_{false}; + microphone_callback_t microphone_callback_{nullptr}; + std::unique_ptr microphone_task_{nullptr}; + i2s_chan_handle_t audio_rx_handle{nullptr}; + std::vector audio_rx_buffer; + // microphone volume (percent), mapped onto the ES8311 analog gain range + std::atomic mic_volume_{70.0f}; + // uSD card std::atomic sd_card_initialized_{false}; sdmmc_card_t *sdcard_{nullptr}; diff --git a/components/esp32-p4-function-ev-board/src/audio.cpp b/components/esp32-p4-function-ev-board/src/audio.cpp index 1f9dd137e..fa1832704 100644 --- a/components/esp32-p4-function-ev-board/src/audio.cpp +++ b/components/esp32-p4-function-ev-board/src/audio.cpp @@ -1,6 +1,7 @@ #include "esp32-p4-function-ev-board.hpp" #include +#include #include #include @@ -43,10 +44,12 @@ bool Esp32P4FunctionEvBoard::initialize_audio(uint32_t sample_rate, set_es8311_write(espp::make_i2c_addressed_write(es8311_i2c_device_)); set_es8311_read(espp::make_i2c_addressed_read_register(es8311_i2c_device_)); - // Create the I2S standard channel for playback (TX). MCLK = 256 * fs (default). + // Create the I2S standard channels: TX for playback, RX for the ES8311's + // ADC (initialized on demand by initialize_microphone(); the codec is full + // duplex on this single bus, sharing the clock). MCLK = 256 * fs (default). i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(audio_i2s_port, I2S_ROLE_MASTER); chan_cfg.auto_clear = true; - if (i2s_new_channel(&chan_cfg, &audio_tx_handle, nullptr) != ESP_OK) { + if (i2s_new_channel(&chan_cfg, &audio_tx_handle, &audio_rx_handle) != ESP_OK) { logger_.error("Failed to create I2S channel"); return false; } @@ -124,26 +127,146 @@ void Esp32P4FunctionEvBoard::mute(bool mute) { bool Esp32P4FunctionEvBoard::is_muted() const { return mute_; } -void Esp32P4FunctionEvBoard::play_audio(const uint8_t *data, uint32_t num_bytes) { +size_t Esp32P4FunctionEvBoard::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!audio_initialized_ || !data || num_bytes == 0) { - return; + return 0; } - // Non-blocking: enqueue as much as currently fits in the TX stream buffer for - // the audio task to drain to I2S. This never blocks the caller, so it is safe - // to call from a touch callback / any task. If the buffer is full the excess - // is dropped (the clip is truncated) rather than stalling the caller. - xStreamBufferSend(audio_tx_stream, data, num_bytes, 0); + // Enqueue only whole 16-bit samples (2 bytes; the TX slot is mono) that + // actually fit: xStreamBufferSend can accept fewer bytes than requested when + // the buffer is nearly full, and a partial (odd) send would strand a byte and + // shift framing on subsequent appends. Cap the request to the free space + // rounded down to a whole sample. This runs in task context, so the non-ISR + // send with a 0 timeout never blocks; the number of bytes actually queued is + // returned so callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 2; + if (sendable == 0) { + return 0; + } + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); +} + +size_t Esp32P4FunctionEvBoard::play_audio(std::span data) { + return play_audio(data.data(), data.size()); +} + +////////////////////////// +// Microphone Functions // +////////////////////////// + +// Map a 0-100% microphone volume onto the ES8311's analog microphone gain +// steps (ES8311_MIC_GAIN_0DB .. ES8311_MIC_GAIN_42DB, 6 dB apart) +static es8311_mic_gain_t microphone_gain_from_volume(float volume) { + int step = static_cast(std::lround(volume / 100.0f * 7.0f)); + step = std::clamp(step, static_cast(ES8311_MIC_GAIN_0DB), + static_cast(ES8311_MIC_GAIN_42DB)); + return static_cast(step); +} + +bool Esp32P4FunctionEvBoard::initialize_microphone(const microphone_callback_t &callback, + const espp::Task::BaseConfig &task_config) { + logger_.info("Initializing microphone"); + if (microphone_initialized_) { + logger_.warn("Microphone already initialized, not initializing again!"); + return false; + } + if (!audio_initialized_) { + logger_.error("The audio subsystem must be initialized first: the ES8311 is a full-duplex " + "codec on a single I2S bus"); + return false; + } + if (!callback) { + logger_.error("A callback is required to receive the recorded audio data"); + return false; + } + microphone_callback_ = callback; + + // The RX channel shares the TX BCLK/WS in full-duplex mode. Receive in + // stereo (both 16-bit slots of every frame) even though the codec's ADC is + // mono: a mono RX slot configuration in this full-duplex setup does not + // deliver one sample per frame (each sample comes through twice, i.e. at + // half speed); capturing both slots gives a deterministic L,R word layout + // and the microphone task keeps only the left slot, where the ES8311 + // drives its ADC data. + i2s_std_config_t rx_cfg = audio_std_cfg; + rx_cfg.slot_cfg = + I2S_STD_PHILIP_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_STEREO); + if (i2s_channel_init_std_mode(audio_rx_handle, &rx_cfg) != ESP_OK) { + logger_.error("Failed to init I2S RX std mode"); + return false; + } + // one update period's worth of stereo frames (NUM_CHANNELS is 2) + audio_rx_buffer.resize(calc_audio_buffer_size(audio_sample_rate())); + if (i2s_channel_enable(audio_rx_handle) != ESP_OK) { + logger_.error("Failed to enable I2S RX channel"); + return false; + } + + // Enable the codec's ADC path alongside the running DAC and apply the + // stored microphone gain + es8311_codec_ctrl_state(AUDIO_HAL_CODEC_MODE_BOTH, AUDIO_HAL_CTRL_START); + es8311_set_mic_gain(microphone_gain_from_volume(mic_volume_)); + + using namespace std::placeholders; + microphone_task_ = espp::Task::make_unique({ + .callback = std::bind(&Esp32P4FunctionEvBoard::microphone_task_callback, this, _1, _2, _3), + .task_config = task_config, + }); + + if (!microphone_task_->start()) { + i2s_channel_disable(audio_rx_handle); + microphone_task_.reset(); + return false; + } + + microphone_initialized_ = true; + return true; +} + +bool Esp32P4FunctionEvBoard::microphone_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. + auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), + &bytes_read, pdMS_TO_TICKS(100)); + if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { + // compact the L,R word pairs down to mono in place, keeping the left + // slot (the ES8311's ADC data) + auto *samples = reinterpret_cast(audio_rx_buffer.data()); + size_t num_frames = bytes_read / (2 * sizeof(int16_t)); + for (size_t i = 0; i < num_frames; i++) { + samples[i] = samples[2 * i]; + } + microphone_callback_(audio_rx_buffer.data(), num_frames * sizeof(int16_t)); + } + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running } -void Esp32P4FunctionEvBoard::play_audio(std::span data) { - play_audio(data.data(), data.size()); +void Esp32P4FunctionEvBoard::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (microphone_initialized_) { + es8311_set_mic_gain(microphone_gain_from_volume(mic_volume_)); + } } +float Esp32P4FunctionEvBoard::microphone_volume() const { return mic_volume_; } + bool Esp32P4FunctionEvBoard::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); + // only ever hand whole 16-bit samples to I2S; a partial sample would shift + // the framing of everything after it + available &= ~static_cast(1); uint8_t *tx_buf = audio_tx_buffer.data(); memset(tx_buf, 0, buffer_size); if (available == 0) { diff --git a/components/m5stack-cardputer/README.md b/components/m5stack-cardputer/README.md index 04955f53f..b428718d6 100644 --- a/components/m5stack-cardputer/README.md +++ b/components/m5stack-cardputer/README.md @@ -24,8 +24,9 @@ singleton hardware abstraction for initializing and using the subsystems: an NS4168 amplifier directly; on the ADV it goes through an ES8311 codec (initialized automatically) into an NS4150B amplifier. - **Microphone**: a recording task delivers 16-bit mono samples to a - callback. Original: SPM1423 PDM microphone; ADV: analog MEMS microphone via - the ES8311 codec's ADC. + callback, with adjustable volume via `microphone_volume()`. Original: + SPM1423 PDM microphone (volume applied in software); ADV: analog MEMS + microphone via the ES8311 codec's ADC (volume applied in the codec). - **uSD card**: SPI-mode micro-SD mounted at `/sdcard`. - **RGB LED**: the StampS3's WS2812 via `espp::Neopixel`. - **Battery**: battery voltage measurement (2:1 divider into ADC1) and an diff --git a/components/m5stack-cardputer/example/README.md b/components/m5stack-cardputer/example/README.md index b55ebb915..744aad43f 100644 --- a/components/m5stack-cardputer/example/README.md +++ b/components/m5stack-cardputer/example/README.md @@ -29,6 +29,8 @@ component to initialize and use the components on the M5Stack Cardputer: | Fn + 2 (F2) | Show / hide the IMU overlay (ADV) | | Fn + 3 (F3) | Start / stop recording from the microphone (ADV) | | Fn + 4 (F4) | Play back / stop the recording (ADV) | +| Fn + 5 / 6 (F5 / F6) | Speaker volume down / up | +| Fn + 7 / 8 (F7 / F8) | Microphone volume down / up (75% = 0 dB) | | Fn + number row | Show F1-F12 in the status bar | | G0 (BOOT) button | Cycle the RGB LED color | diff --git a/components/m5stack-cardputer/example/main/gui.hpp b/components/m5stack-cardputer/example/main/gui.hpp index d51d7d2c4..403334051 100644 --- a/components/m5stack-cardputer/example/main/gui.hpp +++ b/components/m5stack-cardputer/example/main/gui.hpp @@ -41,6 +41,8 @@ class Gui { "fn+2 (F2) show/hide IMU\n" "fn+3 (F3) record / stop\n" "fn+4 (F4) play recording\n" + "fn+5/6 speaker vol -/+\n" + "fn+7/8 mic vol -/+\n" "G0 button cycle LED color\n" "\n" "While help is open:\n" diff --git a/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp b/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp index 24c619943..80c309076 100644 --- a/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp +++ b/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp @@ -154,6 +154,21 @@ extern "C" void app_main(void) { gui.set_status_text("Playing... (fn+4 stops)"); } play_beep(cardputer, 660.0f); + } else if (event.special == espp::M5StackCardputer::SpecialKey::F5 || + event.special == espp::M5StackCardputer::SpecialKey::F6) { + // speaker volume down / up (the beep gives immediate feedback) + float delta = event.special == espp::M5StackCardputer::SpecialKey::F5 ? -10.0f : 10.0f; + cardputer.volume(cardputer.volume() + delta); + gui.set_status_text(fmt::format("Speaker volume: {:.0f}%", cardputer.volume())); + play_beep(cardputer, 660.0f); + } else if (event.special == espp::M5StackCardputer::SpecialKey::F7 || + event.special == espp::M5StackCardputer::SpecialKey::F8) { + // microphone volume down / up (heard on the next recording) + float delta = event.special == espp::M5StackCardputer::SpecialKey::F7 ? -5.0f : 5.0f; + cardputer.microphone_volume(cardputer.microphone_volume() + delta); + gui.set_status_text( + fmt::format("Mic volume: {:.0f}% (75% = 0 dB)", cardputer.microphone_volume())); + play_beep(cardputer, 660.0f); } else if (event.special != espp::M5StackCardputer::SpecialKey::NONE) { gui.handle_special_key(event.special); gui.set_status_text(espp::M5StackCardputer::special_key_name(event.special)); diff --git a/components/m5stack-cardputer/include/m5stack-cardputer.hpp b/components/m5stack-cardputer/include/m5stack-cardputer.hpp index 45782b13c..7a1936ed3 100644 --- a/components/m5stack-cardputer/include/m5stack-cardputer.hpp +++ b/components/m5stack-cardputer/include/m5stack-cardputer.hpp @@ -415,6 +415,7 @@ class M5StackCardputer : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const std::vector &data); /// Play the audio data @@ -425,6 +426,7 @@ class M5StackCardputer : public BaseComponent { /// \note This function is non-blocking and queues the data for the audio /// task to play; to stream data larger than the internal buffer, /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. size_t play_audio(const uint8_t *data, uint32_t num_bytes); ///////////////////////////////////////////////////////////////////////////// @@ -463,6 +465,20 @@ class M5StackCardputer : public BaseComponent { /// \return The microphone sample rate, in Hz uint32_t microphone_sample_rate() const; + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100); 75 is unity (0 dB, + /// the default) + /// \note On the ADV this adjusts the ES8311 codec's digital ADC volume + /// (values above 75 amplify, up to +32 dB at 100); on the original + /// the samples are scaled in software in the microphone task (values + /// above 75 amplify with saturation) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100); 75 is unity + /// (0 dB) + float microphone_volume() const; + ///////////////////////////////////////////////////////////////////////////// // uSD Card ///////////////////////////////////////////////////////////////////////////// @@ -853,6 +869,8 @@ class M5StackCardputer : public BaseComponent { // microphone sample rate (Hz), set by initialize_microphone() std::atomic mic_sample_rate_{0}; + // microphone volume (percent, 75 = unity / 0 dB) + std::atomic mic_volume_{75.0f}; // display std::shared_ptr> display_; diff --git a/components/m5stack-cardputer/src/audio.cpp b/components/m5stack-cardputer/src/audio.cpp index 8c59687d7..d664c7452 100644 --- a/components/m5stack-cardputer/src/audio.cpp +++ b/components/m5stack-cardputer/src/audio.cpp @@ -180,9 +180,9 @@ bool M5StackCardputer::initialize_sound(uint32_t default_audio_rate, bool M5StackCardputer::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { // Queue the next I2S out frame to write - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); uint8_t *buffer = &audio_tx_buffer[0]; memset(buffer, 0, buffer_size); @@ -249,10 +249,22 @@ size_t M5StackCardputer::play_audio(const std::vector &data) { } size_t M5StackCardputer::play_audio(const uint8_t *data, uint32_t num_bytes) { - if (!sound_initialized_) { + // guard against being called before initialize_sound() (audio_tx_stream is + // not valid until then) and against empty input + if (!sound_initialized_ || !data || num_bytes == 0) { + return 0; + } + // Enqueue only whole 16-bit samples (2 bytes; the TX slot is mono) that + // actually fit: xStreamBufferSend can accept fewer bytes than requested when + // the buffer is nearly full, and a partial (odd) send would strand a byte and + // shift framing on subsequent appends. Cap the request to the free space + // rounded down to a whole sample. This runs in task context, so the non-ISR + // send with a 0 timeout never blocks; the number of bytes actually queued is + // returned so callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 2; + if (sendable == 0) { return 0; } - // don't block here; report how much was actually queued so callers can - // stream data larger than the buffer - return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); } diff --git a/components/m5stack-cardputer/src/m5stack-cardputer.cpp b/components/m5stack-cardputer/src/m5stack-cardputer.cpp index 076573db4..0e3ed21b1 100644 --- a/components/m5stack-cardputer/src/m5stack-cardputer.cpp +++ b/components/m5stack-cardputer/src/m5stack-cardputer.cpp @@ -1,4 +1,5 @@ #include +#include #include #include "m5stack-cardputer.hpp" @@ -135,13 +136,14 @@ bool M5StackCardputer::initialize_es8311_microphone() { // (hardware-proven with the esp-box microphone): M5Unified's minimal // sequence uses minimum PGA gain (0x14 = 0x10) and no digital mic gain, // which records at a near-mute level. + const uint8_t adc_volume = static_cast(std::lround(mic_volume_ / 100.0f * 255.0f)); const uint8_t init[][2] = { - {0x0E, 0x02}, // SYSTEM: enable analog PGA / ADC modulator - {0x14, 0x1A}, // SYSTEM: select Mic1p-Mic1n, raised analog PGA gain - {0x15, 0x40}, // ADC: soft-ramp / ALC configuration - {0x16, 0x24}, // ADC: mic digital gain scale - {0x17, 0xBF}, // ADC: volume 0 dB - {0x1C, 0x6A}, // ADC: equalizer bypass + {0x0E, 0x02}, // SYSTEM: enable analog PGA / ADC modulator + {0x14, 0x1A}, // SYSTEM: select Mic1p-Mic1n, raised analog PGA gain + {0x15, 0x40}, // ADC: soft-ramp / ALC configuration + {0x16, 0x24}, // ADC: mic digital gain scale + {0x17, adc_volume}, // ADC: volume (0x00 = -95.5 dB, 0xBF = 0 dB, 0xFF = +32 dB) + {0x1C, 0x6A}, // ADC: equalizer bypass }; return std::all_of(std::begin(init), std::end(init), [this](const auto &entry) { if (!es8311_write(entry[0], entry[1])) { diff --git a/components/m5stack-cardputer/src/microphone.cpp b/components/m5stack-cardputer/src/microphone.cpp index d000927c6..ab5ffdbdd 100644 --- a/components/m5stack-cardputer/src/microphone.cpp +++ b/components/m5stack-cardputer/src/microphone.cpp @@ -1,3 +1,6 @@ +#include +#include + #include "m5stack-cardputer.hpp" using namespace espp; @@ -106,11 +109,25 @@ bool M5StackCardputer::initialize_microphone(const microphone_callback_t &callba uint32_t M5StackCardputer::microphone_sample_rate() const { return mic_sample_rate_; } +void M5StackCardputer::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (variant() == Variant::ADV && microphone_initialized_) { + // the ES8311 scales in hardware; initialize_es8311_microphone() applies + // the stored volume when the microphone is initialized later + es8311_write(0x17, static_cast(std::lround(mic_volume_ / 100.0f * 255.0f))); + } +} + +float M5StackCardputer::microphone_volume() const { return mic_volume_; } + bool M5StackCardputer::microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), - &bytes_read, portMAX_DELAY); + &bytes_read, pdMS_TO_TICKS(100)); if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { if (mic_stereo_capture_) { // compact the L,R word pairs down to mono in place, keeping the left @@ -121,8 +138,27 @@ bool M5StackCardputer::microphone_task_callback(std::mutex &m, std::condition_va samples[i] = samples[2 * i]; } bytes_read = num_frames * sizeof(int16_t); + } else { + // the original's PDM microphone has no hardware gain, so apply the + // microphone volume in software (75% = unity; the ADV scales in the + // codec instead) + float scale = mic_volume_ / 75.0f; + if (scale != 1.0f) { + auto *samples = reinterpret_cast(audio_rx_buffer.data()); + size_t num_samples = bytes_read / sizeof(int16_t); + for (size_t i = 0; i < num_samples; i++) { + auto v = static_cast(samples[i] * scale); + samples[i] = static_cast(std::clamp(v, INT16_MIN, INT16_MAX)); + } + } } microphone_callback_(audio_rx_buffer.data(), bytes_read); } - return false; // don't stop the task + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running } diff --git a/components/m5stack-tab5/example/README.md b/components/m5stack-tab5/example/README.md index 69419de87..49f7b9a0a 100644 --- a/components/m5stack-tab5/example/README.md +++ b/components/m5stack-tab5/example/README.md @@ -9,7 +9,14 @@ This example demonstrates the comprehensive functionality of the M5Stack Tab5 de ### Core Systems - **5" 720p MIPI-DSI Display**: Initialization and brightness control - **GT911 Multi-Touch Controller**: Touch event handling with callbacks -- **Dual Audio System**: ES8388 codec + ES7210 AEC with recording/playback +- **Dual Audio System**: ES8388 codec + ES7210 AEC with recording/playback. + The on-screen audio row (bottom-left) records from the microphones, plays + the recording back through the speaker, and adjusts the speaker / + microphone volumes; the recording buffer prefers PSRAM and the measured + capture rate is logged when a recording stops. Because the speaker is + mono, the recorded stereo (mic 1 / mic 2) is downmixed to both channels + before playback, and the per-channel peak amplitude is logged so you can + tell a silent capture (peak near 0) from a playback issue. - **BMI270 6-axis IMU**: Real-time motion sensing - **Battery Management**: INA226 power monitoring with charging control @@ -17,6 +24,20 @@ This example demonstrates the comprehensive functionality of the M5Stack Tab5 de - **microSD Card**: File system support with SDIO interface - **Real-Time Clock**: RX8130CE with alarm and wake-up features +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_M5STACK_TAB5_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. + ## Hardware Requirements - M5Stack Tab5 development board diff --git a/components/m5stack-tab5/example/main/gui.cpp b/components/m5stack-tab5/example/main/gui.cpp index ea312e8e0..04d2b7379 100644 --- a/components/m5stack-tab5/example/main/gui.cpp +++ b/components/m5stack-tab5/example/main/gui.cpp @@ -1,18 +1,15 @@ +#include #include #include "gui.hpp" void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); - init_label(); - init_gravity_lines(); - init_buttons(); + init_tabview(); + init_draw_tab(); + init_status_tab(); + init_audio_tab(); init_circle_layer(); - // disable scrolling on the screen (so that it doesn't behave weirdly when - // rotated and drawing with your finger) - lv_obj_set_scrollbar_mode(lv_screen_active(), LV_SCROLLBAR_MODE_OFF); - lv_obj_clear_flag(lv_screen_active(), LV_OBJ_FLAG_SCROLLABLE); } void Gui::deinit_ui() { @@ -20,31 +17,72 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &tab5 = espp::M5StackTab5::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, tab5.display_width(), tab5.display_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(0, 0, 0), 0); +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); + status_tab_ = lv_tabview_add_tab(tabview_, "Status"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } -void Gui::init_label() { - label_ = lv_label_create(lv_screen_active()); +void Gui::init_draw_tab() { + // instructions on the left, with the rotate / brightness / clear buttons + // on the right + label_ = lv_label_create(draw_tab_); + lv_label_set_long_mode(label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(label_, lv_pct(70)); lv_label_set_text(label_, ""); lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 0, 0); lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_LEFT, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + int y_offset; + }; + const ButtonSpec buttons[] = { + {&rotate_button_, LV_SYMBOL_REFRESH, 0}, + {&brightness_button_, LV_SYMBOL_EYE_OPEN, 60}, + {&clear_button_, LV_SYMBOL_TRASH, 120}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(draw_tab_); + lv_obj_set_size(*spec.button, 50, 50); + lv_obj_align(*spec.button, LV_ALIGN_TOP_RIGHT, 0, spec.y_offset); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } } -void Gui::init_gravity_lines() { - auto &tab5 = espp::M5StackTab5::get(); +void Gui::init_status_tab() { + status_label_ = lv_label_create(status_tab_); + lv_label_set_long_mode(status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(status_label_, lv_pct(100)); + lv_label_set_text(status_label_, "Waiting for data..."); + lv_obj_align(status_label_, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_set_style_text_align(status_label_, LV_TEXT_ALIGN_LEFT, 0); lv_style_init(&kalman_line_style_); lv_style_set_line_width(&kalman_line_style_, 8); lv_style_set_line_color(&kalman_line_style_, lv_palette_main(LV_PALETTE_BLUE)); lv_style_set_line_rounded(&kalman_line_style_, true); - kalman_line_ = lv_line_create(lv_screen_active()); + kalman_line_ = lv_line_create(status_tab_); kalman_line_points_[0] = {0, 0}; - kalman_line_points_[1] = {tab5.display_width(), tab5.display_height()}; + kalman_line_points_[1] = {0, 0}; lv_line_set_points(kalman_line_, kalman_line_points_, 2); lv_obj_add_style(kalman_line_, &kalman_line_style_, 0); @@ -53,56 +91,92 @@ void Gui::init_gravity_lines() { lv_style_set_line_color(&madgwick_line_style_, lv_palette_main(LV_PALETTE_RED)); lv_style_set_line_rounded(&madgwick_line_style_, true); - madgwick_line_ = lv_line_create(lv_screen_active()); + madgwick_line_ = lv_line_create(status_tab_); madgwick_line_points_[0] = {0, 0}; - madgwick_line_points_[1] = {tab5.display_width(), tab5.display_height()}; + madgwick_line_points_[1] = {0, 0}; lv_line_set_points(madgwick_line_, madgwick_line_points_, 2); lv_obj_add_style(madgwick_line_, &madgwick_line_style_, 0); } -void Gui::init_buttons() { - // a button in the top left which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(rotate_button_, 50, 50); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_LEFT, 0, 0); - lv_obj_t *rotate_label = lv_label_create(rotate_button_); - lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); - lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); - - // a button next to it which cycles the backlight brightness through - // 25/50/75/100% - brightness_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(brightness_button_, 50, 50); - lv_obj_align(brightness_button_, LV_ALIGN_TOP_LEFT, 60, 0); - lv_obj_t *brightness_label = lv_label_create(brightness_button_); - lv_label_set_text(brightness_label, LV_SYMBOL_EYE_OPEN); - lv_obj_align(brightness_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(brightness_button_, event_callback, LV_EVENT_PRESSED, this); - - // a button in the top right which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); - lv_obj_set_size(clear_button_, 50, 50); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 0); - lv_obj_t *clear_label = lv_label_create(clear_button_); - lv_label_set_text(clear_label, LV_SYMBOL_TRASH); - lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); - lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); +void Gui::init_audio_tab() { + // a column: status line, volume line, then the buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 12, 0); + + audio_status_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_status_label_, lv_pct(100)); + lv_label_set_text(audio_status_label_, "Idle"); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 12, 0); + lv_obj_set_style_pad_row(row, 12, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&record_button_, LV_SYMBOL_AUDIO}, {&play_button_, LV_SYMBOL_PLAY}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + {&mic_down_button_, LV_SYMBOL_MINUS}, {&mic_up_button_, LV_SYMBOL_PLUS}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 60, 60); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } + // remember the record / play button labels so set_record_active / + // set_play_active can swap their symbols + record_button_label_ = lv_obj_get_child(record_button_, 0); + play_button_label_ = lv_obj_get_child(play_button_, 0); + // color the volume buttons so the speaker and microphone pairs are + // distinguishable from each other + lv_obj_set_style_bg_color(mic_down_button_, lv_palette_main(LV_PALETTE_TEAL), 0); + lv_obj_set_style_bg_color(mic_up_button_, lv_palette_main(LV_PALETTE_TEAL), 0); } -void Gui::init_circle_layer() { +void Gui::update_audio_label() { auto &tab5 = espp::M5StackTab5::get(); + int speaker_volume = static_cast(tab5.volume()); + int mic_volume = static_cast(tab5.microphone_volume()); + // this is called every GUI tick; only reformat the label when a value + // actually changes (lv_label_set_text_fmt formats a new string each call) + if (speaker_volume == last_speaker_volume_ && mic_volume == last_mic_volume_) { + return; + } + last_speaker_volume_ = speaker_volume; + last_mic_volume_ = mic_volume; + // Pass the LVGL symbols as %s arguments rather than concatenating them into + // the format-string literal: cppcheck cannot expand the LVGL symbol macros + // and flags the literal concatenation as an unknown macro. + lv_label_set_text_fmt(audio_label_, "Speaker %d%% (%s/%s)\nMic %d%% (teal %s/%s)", speaker_volume, + LV_SYMBOL_VOLUME_MID, LV_SYMBOL_VOLUME_MAX, mic_volume, LV_SYMBOL_MINUS, + LV_SYMBOL_PLUS); +} + +void Gui::init_circle_layer() { + // a transparent, click-through overlay above the tabview which shows the + // touch trail (only populated while the Draw tab is active) circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, tab5.display_width(), tab5.display_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); lv_obj_set_style_bg_opa(circle_layer_, LV_OPA_TRANSP, 0); - lv_obj_set_style_border_width(circle_layer_, 0, 0); - lv_obj_set_style_outline_width(circle_layer_, 0, 0); - lv_obj_set_style_shadow_width(circle_layer_, 0, 0); lv_obj_add_event_cb(circle_layer_, draw_circle_layer, LV_EVENT_DRAW_MAIN, this); lv_obj_move_foreground(circle_layer_); } @@ -111,6 +185,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -126,13 +204,59 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_PRESSED: gui->on_pressed(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_pressed(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &tab5 = espp::M5StackTab5::get(); + if (target == record_button_) { + logger_.info("Record button pressed"); + if (record_callback_) { + record_callback_(); + } + return; + } + if (target == play_button_) { + logger_.info("Play button pressed"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + tab5.volume(tab5.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", tab5.volume()); + update_audio_label(); + return; + } + if (target == mic_down_button_ || target == mic_up_button_) { + float delta = target == mic_down_button_ ? -10.0f : 10.0f; + tab5.microphone_volume(tab5.microphone_volume() + delta); + logger_.info("Microphone volume: {:.0f}%", tab5.microphone_volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button pressed"); next_rotation(); @@ -145,21 +269,53 @@ void Gui::on_pressed(lv_event_t *e) { } } +void Gui::set_record_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(record_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_AUDIO); + lv_obj_set_style_bg_color( + record_button_, active ? lv_palette_main(LV_PALETTE_RED) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_play_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(play_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_PLAY); + lv_obj_set_style_bg_color( + play_button_, active ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_audio_status(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(audio_status_label_, std::string(text).c_str()); +} + void Gui::set_label_text(std::string_view text) { std::lock_guard lock(mutex_); lv_label_set_text(label_, std::string(text).c_str()); } +void Gui::set_status_text(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(status_label_, std::string(text).c_str()); +} + +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::next_rotation() { std::lock_guard lock(mutex_); - auto &tab5 = espp::M5StackTab5::get(); clear_circles_impl(); auto rotation = lv_display_get_rotation(lv_display_get_default()); rotation = static_cast((static_cast(rotation) + 1) % 4); lv_display_set_rotation(lv_display_get_default(), rotation); // update the size of the screen-filling objects - lv_obj_set_size(background_, tab5.rotated_display_width(), tab5.rotated_display_height()); - lv_obj_set_size(circle_layer_, tab5.rotated_display_width(), tab5.rotated_display_height()); + int width = lv_display_get_horizontal_resolution(lv_display_get_default()); + int height = lv_display_get_vertical_resolution(lv_display_get_default()); + lv_obj_set_size(tabview_, width, height); + lv_obj_set_size(circle_layer_, width, height); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_invalidate(circle_layer_); } @@ -183,7 +339,6 @@ void Gui::set_madgwick_down(float vx, float vy) { } void Gui::set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, float vy) { - auto &tab5 = espp::M5StackTab5::get(); // remap the vector according to the current display rotation so that the // line always points toward physical "down" auto rotation = lv_display_get_rotation(lv_display_get_default()); @@ -197,9 +352,16 @@ void Gui::set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, fl std::swap(vx, vy); vy = -vy; } - // draw the line from the center of the screen toward "down" - int x0 = tab5.rotated_display_width() / 2; - int y0 = tab5.rotated_display_height() / 2; + // draw the line from the center of the Status tab's content area toward + // "down" (fall back to the display size if the layout hasn't run yet) + int w = lv_obj_get_content_width(status_tab_); + int h = lv_obj_get_content_height(status_tab_); + if (w <= 0 || h <= 0) { + w = lv_display_get_horizontal_resolution(lv_display_get_default()); + h = lv_display_get_vertical_resolution(lv_display_get_default()) - TAB_BAR_HEIGHT; + } + int x0 = w / 2; + int y0 = h / 2; points[0] = {static_cast(x0), static_cast(y0)}; points[1] = {static_cast(x0 + 50 * vx), static_cast(y0 + 50 * vy)}; diff --git a/components/m5stack-tab5/example/main/gui.hpp b/components/m5stack-tab5/example/main/gui.hpp index a6f3c85aa..099ee9511 100644 --- a/components/m5stack-tab5/example/main/gui.hpp +++ b/components/m5stack-tab5/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -22,16 +23,22 @@ /// dispatched through a single static trampoline (event_callback) into /// member functions, keeping all UI logic inside the class. /// -/// For this example the Gui shows: -/// * A label with instructions and live battery / RTC / IMU data -/// * Two lines showing the direction of gravity ("down") as computed by a -/// Kalman filter (blue) and a Madgwick filter (red) -/// * A button (top-left) which rotates the display through 0/90/180/270 -/// * A button (next to it) which cycles the backlight through 25/50/75/100% -/// * A button (top-right) which clears the circles drawn on the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Draw" tab: instructions, plus the rotate, brightness, and clear +/// buttons. Touching the screen while this tab is active draws circles +/// (on a transparent overlay) and plays a click. +/// * "Status" tab: live battery / RTC / IMU text and two lines showing the +/// direction of gravity ("down") as computed by a Kalman filter (blue) +/// and a Madgwick filter (red). +/// * "Audio" tab: record / play buttons (wired to the example via +/// callbacks) and speaker / microphone volume buttons (acting directly on +/// the BSP), with labels showing the audio state and current volumes. class Gui { public: + /// Callback invoked when the record / play buttons are pressed + using audio_button_callback_t = std::function; + /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -50,10 +57,20 @@ class Gui { deinit_ui(); } - /// Set the text of the main label. Thread-safe. + /// Set the instruction text shown on the Draw tab. Thread-safe. /// @param text The text to display void set_label_text(std::string_view text); + /// Set the live battery / RTC / IMU text shown on the Status tab. + /// Thread-safe. + /// @param text The text to display + void set_status_text(std::string_view text); + + /// Whether the Draw tab is currently active (used by the example to only + /// draw circles / play clicks for touches on that tab). Thread-safe. + /// @return True if the Draw tab is the active tab + bool draw_page_active(); + /// Draw a circle at the given screen coordinates, replacing the oldest /// circle if the maximum number are already visible. Thread-safe. /// @param x The x coordinate (screen space) @@ -87,8 +104,34 @@ class Gui { /// @param vy The y component of the gravity vector void set_madgwick_down(float vx, float vy); + /// Set the callback invoked when the record button is pressed + /// @param callback The callback to invoke + void set_record_callback(audio_button_callback_t callback) { + record_callback_ = std::move(callback); + } + + /// Set the callback invoked when the play button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + + /// Show whether a recording is in progress (turns the record button red + /// and changes its symbol to stop). Thread-safe. + /// @param active True while recording + void set_record_active(bool active); + + /// Show whether a playback is in progress (turns the play button green and + /// changes its symbol to stop). Thread-safe. + /// @param active True while playing + void set_play_active(bool active); + + /// Set the status line on the Audio tab (e.g. "Recording...", "Mic + /// unavailable"). Thread-safe. + /// @param text The text to display + void set_audio_status(std::string_view text); + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 50; struct Circle { int x{0}; @@ -101,14 +144,19 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); - void init_label(); - void init_gravity_lines(); - void init_buttons(); + void init_tabview(); + void init_draw_tab(); + void init_status_tab(); + void init_audio_tab(); void init_circle_layer(); + // update the audio volume label from the BSP's current volumes; called + // with the mutex held + void update_audio_label(); + // update the given "down" line from a vector in the unrotated display - // frame, remapping for the current display rotation + // frame, remapping for the current display rotation; called with the + // mutex held void set_down_line(lv_obj_t *line, lv_point_precise_t *points, float vx, float vy); // the LVGL update task: calls lv_task_handler() under the mutex @@ -118,6 +166,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_pressed(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -128,15 +177,36 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *draw_tab_{nullptr}; + lv_obj_t *status_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *label_{nullptr}; + lv_obj_t *status_label_{nullptr}; lv_obj_t *kalman_line_{nullptr}; lv_obj_t *madgwick_line_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *brightness_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *record_button_{nullptr}; + lv_obj_t *record_button_label_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *play_button_label_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *mic_down_button_{nullptr}; + lv_obj_t *mic_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_speaker_volume_{-1}; + int last_mic_volume_{-1}; + lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t record_callback_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + lv_style_t kalman_line_style_; lv_style_t madgwick_line_style_; lv_point_precise_t kalman_line_points_[2]; @@ -154,7 +224,7 @@ class Gui { espp::Task update_task_{ {.callback = [this](auto &m, auto &cv) { return update(m, cv); }, .task_config = { - .name = "gui", .stack_size_bytes = 10 * 1024, .priority = 20, .core_id = 1}}}; + .name = "gui", .stack_size_bytes = 12 * 1024, .priority = 20, .core_id = 1}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp index 532a5e246..56ca14169 100644 --- a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp +++ b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp @@ -7,11 +7,18 @@ * and communication interfaces. */ +#include +#include #include #include +#include +#include #include #include +#include +#include + #include "m5stack-tab5.hpp" #include "gui.hpp" @@ -25,6 +32,21 @@ static std::vector audio_bytes; static bool load_audio(size_t &out_size, size_t &out_sample_rate); static void play_click(espp::M5StackTab5 &tab5); +// Audio recording state (written by the recording callback, read/controlled +// from the GUI button callbacks and main loop). The recorded data is 16-bit +// interleaved stereo at the current audio sample rate. +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 2; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static std::atomic recording_start_us{0}; +static std::atomic recording_last_us{0}; + extern "C" void app_main(void) { espp::Logger logger({.tag = "M5Stack Tab5 Example", .level = espp::Logger::Verbosity::INFO}); logger.info("Starting example!"); @@ -181,6 +203,11 @@ extern "C" void app_main(void) { return; } + // unmute the audio and set the volume to 60% (do this before the GUI is + // created so its volume label shows the right value) + tab5.mute(false); + tab5.volume(60.0f); + // create the GUI: builds the UI (label, buttons, gravity lines, circle // layer) and starts the task which updates LVGL. All of its public methods // are thread-safe, so the touch callback, button callback, and data display @@ -188,8 +215,9 @@ extern "C" void app_main(void) { logger.info("Setting up LVGL UI..."); static Gui gui({.log_level = espp::Logger::Verbosity::INFO}); static const std::string instructions = - fmt::format("\n\n\n\nTouch the screen!\nPress the {} button to clear circles.\nPress the {} " - "button to rotate the display.\nPress the {} button to cycle the brightness.", + fmt::format("Touch the screen to draw!\nPress the {} button to clear circles.\nPress the " + "{} button to rotate the display.\nPress the {} button to cycle the " + "brightness.\nThe Status and Audio tabs show the other subsystems.", LV_SYMBOL_TRASH, LV_SYMBOL_REFRESH, LV_SYMBOL_EYE_OPEN); gui.set_label_text(instructions); @@ -222,14 +250,19 @@ extern "C" void app_main(void) { if (touchpad_data.btn_state) { gui.clear_circles(); } - // if there is a touch point, draw a circle and play a click sound - if (touchpad_data.num_touch_points > 0) { + // if there is a touch point on the Draw tab, draw a circle and play a + // click sound (touches on the other tabs go to their widgets) + if (touchpad_data.num_touch_points > 0 && gui.draw_page_active()) { play_click(tab5); gui.draw_circle(touchpad_data.x, touchpad_data.y, 10); } } }; logger.info("Initializing touch..."); + // NOTE: this example raises the BSP interrupt-task stack size via + // sdkconfig.defaults (CONFIG_M5STACK_TAB5_INTERRUPT_STACK_SIZE=8192); the + // touch controller is read from that task and its error-logging path + // needs more than the 4 KB BSP default. See the example README. if (!tab5.initialize_touch(touch_callback)) { logger.error("Failed to initialize touch!"); return; @@ -247,13 +280,94 @@ extern "C" void app_main(void) { logger.info("Setting audio sample rate to {} Hz", wav_sample_rate); tab5.audio_sample_rate(wav_sample_rate); - // unmute the audio and set the volume to 60% - tab5.mute(false); - tab5.volume(60.0f); - // set the brightness to 75% tab5.brightness(75.0f); + // Keep the analog microphone gain modest: the ES7210 front-end develops a + // high-frequency whine as the analog gain is raised, so the loudness comes + // from the RMS software makeup gain applied to the recording on stop (see + // below) rather than from the analog stage. The mic +/- buttons still adjust + // the analog gain if desired. + tab5.microphone_volume(40.0f); + + // Allocate the recording buffer (16-bit interleaved stereo at the current + // sample rate): prefer PSRAM, fall back to a couple of seconds in internal + // RAM + size_t bytes_per_second = tab5.audio_sample_rate() * 2 * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + gui.set_audio_status("No recording buffer"); + logger.warn("Could not allocate a recording buffer; recording disabled"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz stereo)", recording_capacity / 1024, + recording_capacity / bytes_per_second, tab5.audio_sample_rate()); + } + + // The recording callback appends the recorded stereo frames to the buffer + // and auto-stops when it is full (the main loop notices and updates the + // GUI) + auto record_data_callback = [](const uint8_t *data, size_t length) { + if (!recording) { + return; + } + size_t offset = recording_len; + size_t to_copy = std::min(length, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = esp_timer_get_time(); + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + recording = false; + } + }; + + // The record button toggles recording; the play button toggles playback of + // the recording (streamed to the speaker by the main loop) + gui.set_record_callback([&]() { + if (recording_capacity == 0) { + logger.warn("Recording unavailable (no buffer)"); + gui.set_audio_status("Recording unavailable (see log)"); + return; + } + if (recording) { + recording = false; // the main loop stops the BSP recording and logs + } else { + playing = false; + gui.set_play_active(false); + recording_len = 0; + recording_start_us = esp_timer_get_time(); + recording_last_us = recording_start_us.load(); + recording = true; + tab5.start_audio_recording(record_data_callback); + gui.set_record_active(true); + gui.set_audio_status("Recording..."); + } + }); + gui.set_play_callback([&]() { + if (playing) { + playing = false; + gui.set_play_active(false); + gui.set_audio_status("Playback stopped"); + } else if (recording_len > 0) { + recording = false; + playing = true; + gui.set_play_active(true); + gui.set_audio_status("Playing..."); + } else { + logger.info("Nothing recorded yet; press the record button first"); + gui.set_audio_status("Nothing recorded yet"); + } + }); + // make a task to read out various data such as IMU, battery monitoring, etc. // and print it to screen logger.info("Starting data display task..."); @@ -262,7 +376,7 @@ extern "C" void app_main(void) { // sleep first in case we don't get IMU data and need to exit early { std::unique_lock lock(m); - cv.wait_for(lock, 20ms); + cv.wait_for(lock, 10ms); } static auto &tab5 = espp::M5StackTab5::get(); static auto imu = tab5.imu(); @@ -331,14 +445,14 @@ extern "C" void app_main(void) { vx = -vx; vy = -vy; - std::string text = fmt::format("{}\n\n\n\n\n", instructions); + std::string text; text += battery_text; text += rtc_text; text += imu_text; // update the GUI with the new data; the Gui handles remapping the // vectors for the current display rotation - gui.set_label_text(text); + gui.set_status_text(text); gui.set_kalman_down(gravity_vector.x, gravity_vector.y); gui.set_madgwick_down(vx, vy); @@ -352,9 +466,83 @@ extern "C" void app_main(void) { }}); imu_task.start(); - // loop forever + // Main loop: stream any active playback to the speaker and notice when a + // recording stops (either button press or the buffer filling up) + size_t play_offset = 0; + bool was_recording = false; while (true) { - std::this_thread::sleep_for(1s); + // feed the active playback in chunks, advancing by however much the + // speaker's stream buffer accepted + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_play_active(false); + gui.set_audio_status("Playback done"); + logger.info("Playback done"); + } else { + play_offset += tab5.play_audio(recording_buffer + play_offset, + std::min(len - play_offset, 16384)); + } + } else { + play_offset = 0; + } + // notice when the recording stopped (button press or buffer full) + bool now_recording = recording; + if (was_recording && !now_recording) { + tab5.stop_audio_recording(); + gui.set_record_active(false); + gui.set_audio_status(fmt::format("Recorded {:.1f}s ({} plays)", + static_cast(recording_len) / + (tab5.audio_sample_rate() * 2 * sizeof(int16_t)), + LV_SYMBOL_PLAY)); + // report the measured capture rate: stereo frames recorded over the + // wall clock they took to arrive should match the nominal sample rate + size_t num_frames = recording_len / (2 * sizeof(int16_t)); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_frames / elapsed_s : 0.0f; + // Post-process the recording for playback on the Tab5's MONO speaker. + // The ES7210 records 16-bit interleaved stereo (mic 1 -> left slot, mic 2 + // -> right); downmix each frame to the average (the speaker plays one I2S + // slot, so the result is written to both). The captured level is low, so + // rather than driving the analog gain hot (which whines), remove the DC + // offset and apply an RMS-normalized software makeup gain: this makes the + // recording play back at a consistent, audible level comparable to the + // click WAV, without a high analog mic gain or a high speaker volume. + auto *samples = reinterpret_cast(recording_buffer); + int16_t peak = 0; + int64_t sum = 0; + for (size_t i = 0; i < num_frames; i++) { + int32_t mono = (static_cast(samples[2 * i]) + samples[2 * i + 1]) / 2; + samples[2 * i] = static_cast(mono); + peak = std::max(peak, static_cast(std::abs(mono))); + sum += mono; + } + int32_t dc = num_frames ? static_cast(sum / static_cast(num_frames)) : 0; + int64_t sum_sq = 0; + for (size_t i = 0; i < num_frames; i++) { + int32_t v = samples[2 * i] - dc; + sum_sq += static_cast(v) * v; + } + double rms = num_frames ? std::sqrt(static_cast(sum_sq) / num_frames) : 0.0; + // target ~-15 dBFS leaves headroom; cap the gain so a near-silent capture + // is not blown up into noise + static constexpr double target_rms = 5500.0; + double gain = rms > 1.0 ? std::clamp(target_rms / rms, 1.0, 64.0) : 1.0; + for (size_t i = 0; i < num_frames; i++) { + int32_t v = static_cast((samples[2 * i] - dc) * gain); + int16_t mono = static_cast(std::clamp(v, -32768, 32767)); + samples[2 * i] = mono; + samples[2 * i + 1] = mono; + } + logger.info("Recorded {} frames in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal); " + "peak={} dc={} rms={:.0f}; applied makeup gain {:.1f}x", + num_frames, elapsed_s, effective_hz, tab5.audio_sample_rate(), peak, dc, rms, + gain); + } + was_recording = now_recording; + std::this_thread::sleep_for(50ms); } //! [m5stack tab5 example] } @@ -388,7 +576,24 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::M5StackTab5 &tab5) { - if (audio_bytes.size() > 0) { - tab5.play_audio(audio_bytes); + // Enqueue the click without blocking the caller (this runs in the touch + // callback). play_audio() enqueues only whole frames and returns how many + // bytes it took, so advance by that count (advancing by the requested size + // would skip samples). Stop as soon as the stream buffer is full rather than + // waiting for it to drain - blocking here would freeze the touch task for the + // whole click. The click comfortably fits in the stream buffer, so it plays + // in full in practice. + if (audio_bytes.empty()) { + return; + } + auto audio_buffer_size = tab5.audio_buffer_size(); + size_t offset = 0; + while (offset < audio_bytes.size()) { + size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); + size_t queued = tab5.play_audio(audio_bytes.data() + offset, chunk); + offset += queued; + if (queued < chunk) { + break; // stream buffer full for now; do not block the caller + } } } diff --git a/components/m5stack-tab5/example/sdkconfig.defaults b/components/m5stack-tab5/example/sdkconfig.defaults index a48e51abc..afef7c936 100644 --- a/components/m5stack-tab5/example/sdkconfig.defaults +++ b/components/m5stack-tab5/example/sdkconfig.defaults @@ -36,7 +36,10 @@ CONFIG_FREERTOS_USE_TRACE_FACILITY=y CONFIG_FREERTOS_USE_STATS_FORMATTING_FUNCTIONS=y # M5Stack Tab5 BSP configuration -CONFIG_M5STACK_TAB5_INTERRUPT_STACK_SIZE=4096 +# Raised from the 4 KB BSP default: the interrupt task runs the touch- +# controller I2C reads and, on a bus error, logs through espp's fmt +# color-print path, which needs more than 4 KB of stack. See the README. +CONFIG_M5STACK_TAB5_INTERRUPT_STACK_SIZE=8192 CONFIG_M5STACK_TAB5_AUDIO_TASK_STACK_SIZE=8192 # PSRAM configuration diff --git a/components/m5stack-tab5/include/m5stack-tab5.hpp b/components/m5stack-tab5/include/m5stack-tab5.hpp index 00710050b..e9c34c7c9 100755 --- a/components/m5stack-tab5/include/m5stack-tab5.hpp +++ b/components/m5stack-tab5/include/m5stack-tab5.hpp @@ -310,22 +310,48 @@ class M5StackTab5 : public BaseComponent { size_t audio_buffer_size() const; /// Play audio data - /// \param data The audio data to play + /// \param data The audio data to play (16-bit signed interleaved stereo) /// \param num_bytes The number of bytes to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. + size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Play audio data - /// \param data The audio data to play - void play_audio(std::span data); + /// \param data The audio data to play (16-bit signed interleaved stereo) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. + size_t play_audio(std::span data); /// Start recording audio - /// \param callback Function to call with recorded audio data + /// \param callback Function to call with recorded audio data: 16-bit + /// signed interleaved stereo (ES7210 microphone 1 on the left slot, + /// microphone 2 on the right) at audio_sample_rate() /// \return True if recording started successfully + /// \note The callback runs in the dedicated microphone task's context (a + /// separate task from audio playback), so keep it quick and non- + /// blocking and size that task's stack accordingly bool start_audio_recording(std::function callback); /// Stop recording audio void stop_audio_recording(); + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100), mapped onto the + /// ES7210 analog microphone gain range (0 dB - +37.5 dB) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100) + float microphone_volume() const; + ///////////////////////////////////////////////////////////////////////////// // IMU & Sensors ///////////////////////////////////////////////////////////////////////////// @@ -504,6 +530,7 @@ class M5StackTab5 : public BaseComponent { bool update_touch(); bool update_battery_status(); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); // Hardware pin definitions based on Tab5 specifications @@ -728,10 +755,13 @@ class M5StackTab5 : public BaseComponent { // Audio system std::atomic audio_initialized_{false}; std::atomic volume_{50.0f}; + // microphone volume (percent), mapped onto the ES7210 analog gain range + std::atomic mic_volume_{70.0f}; std::atomic mute_{false}; std::shared_ptr> es8388_i2c_device_; std::shared_ptr> es7210_i2c_device_; std::unique_ptr audio_task_{nullptr}; + std::unique_ptr microphone_task_{nullptr}; i2s_chan_handle_t audio_tx_handle{nullptr}; i2s_chan_handle_t audio_rx_handle{nullptr}; i2s_std_config_t audio_std_cfg{}; diff --git a/components/m5stack-tab5/src/audio.cpp b/components/m5stack-tab5/src/audio.cpp index 621e82f32..501388596 100644 --- a/components/m5stack-tab5/src/audio.cpp +++ b/components/m5stack-tab5/src/audio.cpp @@ -1,3 +1,6 @@ +#include +#include + #include "m5stack-tab5.hpp" //////////////////////// @@ -6,6 +9,40 @@ namespace espp { +// Map a 0-100% microphone volume onto the ES7210's analog gain steps +// (GAIN_0DB .. GAIN_37_5DB) +static es7210_gain_value_t microphone_gain_from_volume(float volume) { + int step = static_cast(std::lround(volume / 100.0f * static_cast(GAIN_37_5DB))); + step = std::clamp(step, static_cast(GAIN_0DB), static_cast(GAIN_37_5DB)); + return static_cast(step); +} + +// Map a PCM sample rate onto the codec's audio_hal sample-rate enum. The ES7210 +// coefficient table is selected by this enum, so it must match the rate the I2S +// clocks actually generate; an unsupported rate falls back to 48 kHz. +static audio_hal_iface_samples_t audio_hal_samples_from_rate(uint32_t rate) { + switch (rate) { + case 8000: + return AUDIO_HAL_08K_SAMPLES; + case 11025: + return AUDIO_HAL_11K_SAMPLES; + case 16000: + return AUDIO_HAL_16K_SAMPLES; + case 22050: + return AUDIO_HAL_22K_SAMPLES; + case 24000: + return AUDIO_HAL_24K_SAMPLES; + case 32000: + return AUDIO_HAL_32K_SAMPLES; + case 44100: + return AUDIO_HAL_44K_SAMPLES; + case 48000: + return AUDIO_HAL_48K_SAMPLES; + default: + return AUDIO_HAL_48K_SAMPLES; + } +} + bool M5StackTab5::initialize_audio(uint32_t sample_rate, const espp::Task::BaseConfig &task_config) { logger_.info("Initializing dual audio system (ES8388 + ES7210) at {} Hz", sample_rate); @@ -38,38 +75,14 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, logger_.info("Configuring I2S standard mode with sample rate {} Hz", sample_rate); ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_tx_handle, &audio_std_cfg)); - // Optional RX channel for recording (ES7210) - logger_.info("Creating I2S channel for recording (RX)"); - i2s_tdm_config_t tdm_cfg = { - .clk_cfg = - { - .sample_rate_hz = (uint32_t)48000, - .clk_src = I2S_CLK_SRC_DEFAULT, - .ext_clk_freq_hz = 0, - .mclk_multiple = I2S_MCLK_MULTIPLE_256, - .bclk_div = 8, - }, - .slot_cfg = {.data_bit_width = I2S_DATA_BIT_WIDTH_16BIT, - .slot_bit_width = I2S_SLOT_BIT_WIDTH_AUTO, - .slot_mode = I2S_SLOT_MODE_STEREO, - .slot_mask = (i2s_tdm_slot_mask_t)(I2S_TDM_SLOT0 | I2S_TDM_SLOT1 | - I2S_TDM_SLOT2 | I2S_TDM_SLOT3), - .ws_width = I2S_TDM_AUTO_WS_WIDTH, - .ws_pol = false, - .bit_shift = true, - .left_align = false, - .big_endian = false, - .bit_order_lsb = false, - .skip_mask = false, - .total_slot = I2S_TDM_AUTO_SLOT_NUM}, - .gpio_cfg = {.mclk = audio_mclk_io, - .bclk = audio_sclk_io, - .ws = audio_lrck_io, - .dout = audio_dsdin_io, // ES8388 DSDIN (playback data input) - .din = audio_asdout_io, // ES7210 ASDOUT (record data output) - .invert_flags = {.mclk_inv = false, .bclk_inv = false, .ws_inv = false}}, - }; - ESP_ERROR_CHECK(i2s_channel_init_tdm_mode(audio_rx_handle, &tdm_cfg)); + // RX channel for recording (ES7210). The ES7210 is configured below for + // standard I2S output (16-bit stereo: microphone 1 on the left slot, + // microphone 2 on the right), and in full-duplex mode the RX module shares + // the TX BCLK/WS, so the RX channel must use the same standard-mode + // configuration - a different frame geometry (e.g. multi-slot TDM) would + // not match the shared clock. + logger_.info("Configuring I2S channel for recording (RX)"); + ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_rx_handle, &audio_std_cfg)); // Stream buffers and task auto tx_buf_size = calc_audio_buffer_size(sample_rate); @@ -152,7 +165,9 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, es7210_cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; es7210_cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; es7210_cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; - es7210_cfg.i2s_iface.samples = AUDIO_HAL_48K_SAMPLES; + // configure the ES7210 for the rate the I2S clocks are actually running at + // (rather than assuming 48 kHz) so its coefficient table matches + es7210_cfg.i2s_iface.samples = audio_hal_samples_from_rate(sample_rate); logger_.info("Initializing ES7210 codec for recording (ADC) at {} Hz", sample_rate); if (es7210_adc_init(&es7210_cfg) != ESP_OK) { logger_.error("ES7210 init failed"); @@ -161,21 +176,75 @@ bool M5StackTab5::initialize_audio(uint32_t sample_rate, if (es7210_adc_config_i2s(AUDIO_HAL_CODEC_MODE_ENCODE, &es7210_cfg.i2s_iface) != ESP_OK) { logger_.error("ES7210 I2S cfg failed"); } + // Enable the ES7210 digital high-pass filter on the ADC channels to strip + // the microphone DC offset. The espp es7210 driver leaves these registers + // at their reset value (HPF off), so the DC offset is amplified by the + // analog gain and rails the ADC to full scale - recordings come out as a + // near-constant DC level that the AC-coupled speaker plays as a click then + // silence. These are the driver's documented "quick setup" HPF values. + // write_register clears the error code on success, so a later successful + // write would mask an earlier failure; remember the first failure explicitly. + std::error_code hpf_ec, first_hpf_ec; + auto write_hpf = [&](uint8_t reg, uint8_t val) { + es7210_i2c_device_->write_register(reg, std::vector{val}, hpf_ec); + if (hpf_ec && !first_hpf_ec) { + first_hpf_ec = hpf_ec; + } + }; + write_hpf(0x22, 0x0a); // ADC1/2 HPF1 + write_hpf(0x23, 0x2a); // ADC1/2 HPF2 + write_hpf(0x20, 0x0a); // ADC3/4 HPF2 + write_hpf(0x21, 0x2a); // ADC3/4 HPF1 + if (first_hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", first_hpf_ec.message()); + } + // apply the stored microphone volume (the driver's init leaves the analog + // gain at its 0 dB minimum, which records very quietly) + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); es7210_adc_ctrl_state(AUDIO_HAL_CODEC_MODE_ENCODE, AUDIO_HAL_CTRL_START); - // now make the audio task - logger_.info("Creating audio task for playback and recording"); + // Create two independent tasks, matching the (well-behaved) esp-box audio + // path: one drains the playback stream buffer to the TX channel, and a + // separate task blocks on the RX channel to read the microphone. Keeping the + // record read off the playback task means the RX DMA ring is drained at + // exactly the rate samples arrive (rather than only between blocking TX + // writes), which the shared-task approach did irregularly - on this + // full-duplex controller that jitter degraded the recording and let the ring + // back up. + logger_.info("Creating audio playback and microphone tasks"); using namespace std::placeholders; audio_task_ = espp::Task::make_unique( {.callback = std::bind(&M5StackTab5::audio_task_callback, this, _1, _2, _3), .task_config = task_config}); + // give the microphone task a distinct name from the playback task (they share + // the rest of the caller's task config) so the two are easy to tell apart in + // logs and while debugging + auto mic_task_config = task_config; + mic_task_config.name = + task_config.name.empty() ? std::string("microphone") : task_config.name + " microphone"; + microphone_task_ = espp::Task::make_unique( + {.callback = std::bind(&M5StackTab5::microphone_task_callback, this, _1, _2, _3), + .task_config = mic_task_config}); + + // Start the tasks stepwise so a failure does not leave a half-initialized + // state: if the microphone task fails to start, stop the already-running + // playback task and report failure rather than leaving it running with + // audio_initialized_ set. + if (!audio_task_->start()) { + logger_.error("Failed to start the audio playback task"); + return false; + } + if (!microphone_task_->start()) { + logger_.error("Failed to start the microphone task"); + audio_task_->stop(); + return false; + } // Enable speaker output enable_audio(true); audio_initialized_ = true; - - return audio_task_->start(); + return true; } void M5StackTab5::enable_audio(bool enable) { @@ -200,26 +269,58 @@ void M5StackTab5::mute(bool mute) { bool M5StackTab5::is_muted() const { return mute_; } -void M5StackTab5::play_audio(const uint8_t *data, uint32_t num_bytes) { +size_t M5StackTab5::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!audio_initialized_ || !data || num_bytes == 0) { - return; + return 0; } - // Don't block here: append what fits into the stream buffer and return - // immediately. The audio task drains it to the I2S peripheral. Matches the - // esp-box / t-deck playback path. - xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); + // Enqueue only whole 16-bit stereo frames (4 bytes) that actually fit: + // xStreamBufferSend can accept fewer bytes than requested when the buffer is + // nearly full, and a partial (non-frame-aligned) send would strand 1-3 bytes + // and corrupt the L/R framing of subsequent appends. Cap the request to the + // free space rounded down to a whole frame so the send is always + // frame-aligned. This runs in task context, so the non-ISR send with a 0 + // timeout never blocks. The number of bytes actually queued is returned so + // callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 4; + if (sendable == 0) { + return 0; + } + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); +} + +size_t M5StackTab5::play_audio(std::span data) { + return play_audio(data.data(), data.size()); } -void M5StackTab5::play_audio(std::span data) { - play_audio(data.data(), data.size()); +void M5StackTab5::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (audio_initialized_) { + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + } } +float M5StackTab5::microphone_volume() const { return mic_volume_; } + bool M5StackTab5::start_audio_recording( std::function callback) { if (!audio_initialized_) { logger_.error("Audio system not initialized"); return false; } + // Reject a start while a recording is already in progress: the microphone + // task reads audio_rx_callback_ whenever recording_ is true, so reassigning + // the callback here (without stopping first) would race that task. The caller + // must stop_audio_recording() before starting a new one. + if (recording_) { + logger_.warn("Recording already in progress; stop it before starting a new one"); + return false; + } + // The microphone task drains the RX ring continuously, so there is no stale + // backlog to flush here (and flushing would race that task on the same RX + // handle). Install the callback before arming recording_: the mic task only + // reads the callback once it observes recording_ == true, and the release/ + // acquire on that atomic publishes the callback write to it. audio_rx_callback_ = callback; recording_ = true; logger_.info("Audio recording started"); @@ -237,9 +338,9 @@ bool M5StackTab5::audio_task_callback(std::mutex &m, std::condition_variable &cv // Queue the next I2S out frame to write (matches the esp-box / t-deck path): // always write a full, frame-aligned buffer with the queued samples zero- // padded up to buffer_size so the I2S DMA is fed at a constant cadence. - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); uint8_t *tx_buf = audio_tx_buffer.data(); memset(tx_buf, 0, buffer_size); if (available == 0) { @@ -249,18 +350,35 @@ bool M5StackTab5::audio_task_callback(std::mutex &m, std::condition_variable &cv i2s_channel_write(audio_tx_handle, tx_buf, buffer_size, NULL, portMAX_DELAY); } - // Recording: read from RX channel and invoke callback - if (recording_) { - size_t bytes_read = 0; - i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), &bytes_read, - 0); - if (bytes_read > 0 && audio_rx_callback_) { - audio_rx_callback_(audio_rx_buffer.data(), bytes_read); - } - } return false; } +bool M5StackTab5::microphone_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + // Block until the RX channel has a buffer of microphone samples. This runs on + // its own task so the ES7210's RX DMA ring is always drained at the rate + // samples arrive - in full-duplex RX and TX share one I2S controller, so + // letting the ring back up perturbs the TX (pops the playback). Only forward + // the samples to the callback while a recording is in progress; otherwise the + // read still drains the ring and the data is discarded. + size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. + auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), + &bytes_read, pdMS_TO_TICKS(100)); + if (err == ESP_OK && bytes_read > 0 && recording_ && audio_rx_callback_) { + audio_rx_callback_(audio_rx_buffer.data(), bytes_read); + } + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running +} + uint32_t M5StackTab5::audio_sample_rate() const { return audio_std_cfg.clk_cfg.sample_rate_hz; } size_t M5StackTab5::audio_buffer_size() const { return audio_tx_buffer.size(); } diff --git a/components/m5stack-tab5/src/m5stack-tab5.cpp b/components/m5stack-tab5/src/m5stack-tab5.cpp index b7fb4e038..ae554cf46 100644 --- a/components/m5stack-tab5/src/m5stack-tab5.cpp +++ b/components/m5stack-tab5/src/m5stack-tab5.cpp @@ -21,7 +21,7 @@ bool M5StackTab5::initialize_io_expanders() { .device_address = 0x43, .timeout_ms = static_cast(internal_i2c_.config().timeout_ms), .scl_speed_hz = internal_i2c_.config().clk_speed, - .log_level = Logger::Verbosity::INFO, + .log_level = Logger::Verbosity::WARN, }, ec); if (!ioexp_0x43_i2c_device_) { @@ -33,7 +33,7 @@ bool M5StackTab5::initialize_io_expanders() { .device_address = 0x44, .timeout_ms = static_cast(internal_i2c_.config().timeout_ms), .scl_speed_hz = internal_i2c_.config().clk_speed, - .log_level = Logger::Verbosity::INFO, + .log_level = Logger::Verbosity::WARN, }, ec); if (!ioexp_0x44_i2c_device_) { diff --git a/components/smartpanlee-sc01-plus/example/README.md b/components/smartpanlee-sc01-plus/example/README.md index caca7ca07..069aac02d 100755 --- a/components/smartpanlee-sc01-plus/example/README.md +++ b/components/smartpanlee-sc01-plus/example/README.md @@ -10,12 +10,28 @@ BSP component on the Smart Panlee SC01 Plus touchscreen display module. - ST7796 display initialization over the ESP32-S3 8-bit parallel LCD bus - FT5x06 touch handling with LVGL integration - Foreground custom-drawn touch trail with bounded invalidation for smooth redraws -- I2S speaker playback with a bundled touch-click sound +- I2S speaker playback with a bundled touch-click sound, plus an on-screen + audio row (bottom-left) to play the sound, toggle mute, and adjust the + volume - Backlight brightness control - Optional microSD mounting and filesystem inspection - Published peripheral pin maps for I2S, RS-485, and external GPIOs - Rotation-aware UI layout with wrapped instructions that stay on-screen in portrait/landscape +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_SMARTPANLEE_SC01_PLUS_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. + ## Hardware Required - Smart Panlee SC01 Plus / WT32-SC01-PLUS compatible board diff --git a/components/smartpanlee-sc01-plus/example/main/gui.cpp b/components/smartpanlee-sc01-plus/example/main/gui.cpp index a86989878..607b95a28 100644 --- a/components/smartpanlee-sc01-plus/example/main/gui.cpp +++ b/components/smartpanlee-sc01-plus/example/main/gui.cpp @@ -4,9 +4,10 @@ void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); + init_tabview(); init_label(); init_buttons(); + init_audio_controls(); init_circle_layer(); // disable scrolling on the screen (so that it doesn't behave weirdly when // rotated and drawing with your finger) @@ -19,41 +20,46 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &board = espp::SmartPanleeSc01Plus::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, board.rotated_display_width(), board.rotated_display_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(8, 12, 24), 0); +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } void Gui::init_label() { - auto &board = espp::SmartPanleeSc01Plus::get(); - label_ = lv_label_create(lv_screen_active()); - lv_label_set_text(label_, ""); + label_ = lv_label_create(draw_tab_); lv_label_set_long_mode(label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(label_, lv_pct(75)); + lv_label_set_text(label_, ""); + lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 0, 0); lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_LEFT, 0); - // size / align the label for the current display width, leaving room on - // the right for the buttons - auto label_width = std::max(static_cast(board.rotated_display_width()) - 96, 120); - lv_obj_set_width(label_, label_width); - lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 16, 16); } void Gui::init_buttons() { - // a button in the top right which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); + // rotate / clear buttons in the top right of the Draw tab + rotate_button_ = lv_btn_create(draw_tab_); lv_obj_set_size(rotate_button_, 56, 56); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, -12, 12); + lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, 0, 0); lv_obj_t *rotate_label = lv_label_create(rotate_button_); lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); - // a button below the rotate button which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); + clear_button_ = lv_btn_create(draw_tab_); lv_obj_set_size(clear_button_, 56, 56); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, -12, 80); + lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 66); lv_obj_add_state(clear_button_, LV_STATE_CHECKED); // make the button red lv_obj_t *clear_label = lv_label_create(clear_button_); lv_label_set_text(clear_label, LV_SYMBOL_TRASH); @@ -61,11 +67,65 @@ void Gui::init_buttons() { lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); } -void Gui::init_circle_layer() { +void Gui::init_audio_controls() { + // the Audio tab: a column with the volume line, then play / mute / volume + // buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 12, 0); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 12, 0); + lv_obj_set_style_pad_row(row, 12, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&play_button_, LV_SYMBOL_PLAY}, + {&mute_button_, LV_SYMBOL_MUTE}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, + {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 56, 56); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } +} + +void Gui::update_audio_label() { auto &board = espp::SmartPanleeSc01Plus::get(); + int volume = static_cast(board.volume()); + bool muted = board.is_muted(); + if (volume == last_volume_ && muted == last_muted_) { + return; + } + last_volume_ = volume; + last_muted_ = muted; + // pass the LVGL symbol as a %s argument rather than concatenating it into the + // format-string literal: cppcheck cannot expand the LVGL symbol macros and + // flags the literal concatenation as an unknown macro + lv_label_set_text_fmt(audio_label_, "%s %d%%%s", LV_SYMBOL_VOLUME_MAX, volume, + muted ? " (muted)" : ""); +} + +void Gui::init_circle_layer() { circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, board.rotated_display_width(), board.rotated_display_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); @@ -81,14 +141,7 @@ void Gui::update_layout() { auto &board = espp::SmartPanleeSc01Plus::get(); int width = board.rotated_display_width(); int height = board.rotated_display_height(); - lv_obj_set_size(background_, width, height); - // size / align the label for the new display width, leaving room on the - // right for the buttons - auto label_width = std::max(width - 96, 120); - lv_obj_set_width(label_, label_width); - lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 16, 16); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, -12, 12); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, -12, 80); + lv_obj_set_size(tabview_, width, height); lv_obj_set_size(circle_layer_, width, height); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_move_foreground(circle_layer_); @@ -99,6 +152,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -114,13 +171,51 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_PRESSED: gui->on_pressed(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_pressed(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &board = espp::SmartPanleeSc01Plus::get(); + if (target == play_button_) { + logger_.info("Play button pressed"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == mute_button_) { + board.mute(!board.is_muted()); + logger_.info("Muted: {}", board.is_muted()); + update_audio_label(); + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + board.volume(board.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", board.volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button pressed"); next_rotation(); @@ -130,6 +225,11 @@ void Gui::on_pressed(lv_event_t *e) { } } +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::set_label_text(std::string_view text) { std::lock_guard lock(mutex_); lv_label_set_text(label_, std::string(text).c_str()); diff --git a/components/smartpanlee-sc01-plus/example/main/gui.hpp b/components/smartpanlee-sc01-plus/example/main/gui.hpp index 8f751bcd4..368976e9b 100644 --- a/components/smartpanlee-sc01-plus/example/main/gui.hpp +++ b/components/smartpanlee-sc01-plus/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -23,13 +24,18 @@ /// member functions, keeping all UI logic inside the class. /// /// For this example the Gui shows: -/// * A wrapping info label (top-left) with instructions -/// * A button (top-right) which rotates the display through 0/90/180/270 -/// * A button (below the rotate button) which clears the circles drawn on -/// the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Draw" tab: instructions, plus the rotate and clear buttons. Touching +/// the screen while this tab is active draws circles (on a transparent +/// overlay) and plays a click. +/// * "Audio" tab: a play-sound button (wired to the example via a +/// callback), a mute toggle, and volume down / up buttons (acting +/// directly on the BSP), with a label showing the current volume. class Gui { public: + /// Callback invoked when the play-sound button is pressed + using audio_button_callback_t = std::function; /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -66,8 +72,18 @@ class Gui { /// re-aligning the UI to match. Thread-safe. void next_rotation(); + /// Whether the Draw tab is currently active (used by the example to only + /// draw circles / play clicks for touches on that tab). Thread-safe. + /// @return True if the Draw tab is the active tab + bool draw_page_active(); + + /// Set the callback invoked when the play-sound button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 44; struct Circle { int x{0}; @@ -80,11 +96,16 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); + void init_tabview(); void init_label(); void init_buttons(); + void init_audio_controls(); void init_circle_layer(); + // update the audio volume label from the BSP's current volume / mute + // state; called with the mutex held + void update_audio_label(); + // re-size / re-align the UI to fill the (possibly rotated) display void update_layout(); @@ -95,6 +116,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_pressed(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -105,18 +127,33 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *draw_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *label_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *mute_button_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_volume_{-1}; + bool last_muted_{false}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + std::array circles_; size_t next_circle_index_{0}; size_t visible_circle_count_{0}; espp::Task update_task_{{.callback = [this](auto &m, auto &cv) { return update(m, cv); }, - .task_config = {.name = "gui", .stack_size_bytes = 6 * 1024}}}; + // NOTE: rendering the tabview (nested containers + flex layout) uses + // noticeably more stack than a flat UI; 6 KB overflows + .task_config = {.name = "gui", .stack_size_bytes = 12 * 1024}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp b/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp old mode 100755 new mode 100644 index 0a47e4e34..772bd65df --- a/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp +++ b/components/smartpanlee-sc01-plus/example/main/smartpanlee_sc01_plus_example.cpp @@ -67,11 +67,15 @@ extern "C" void app_main(void) { // thread-safe, so the touch callback below can call them directly. static Gui gui({.log_level = espp::Logger::Verbosity::INFO}); gui.set_label_text( - fmt::format("Smart Panlee SC01 Plus\n\nTouch the screen to draw and play a click.\nPress {} " - "to rotate.\nPress {} to clear.\nCheck serial output for SD card, pin, and " - "audio info.", + fmt::format("Smart Panlee SC01 Plus\n\nTouch the screen to draw and play a click.\nPress " + "{} to rotate.\nPress {} to clear.\nThe Audio tab plays the click sound and " + "adjusts / mutes the volume.\nCheck serial output for SD card, pin, and audio " + "info.", LV_SYMBOL_REFRESH, LV_SYMBOL_TRASH)); + // the play button on the audio row plays the same click sound as a touch + gui.set_play_callback([&]() { play_click(board); }); + // initialize the touchpad after the GUI exists so touch events can update // it immediately; each touch draws a circle and plays a click sound auto touch_callback = [&](const auto &touch) { @@ -79,12 +83,16 @@ extern "C" void app_main(void) { auto touchpad_data = board.touchpad_convert(touch); if (touchpad_data != previous_touchpad_data) { previous_touchpad_data = touchpad_data; - if (touchpad_data.num_touch_points > 0) { + if (touchpad_data.num_touch_points > 0 && gui.draw_page_active()) { play_click(board); gui.draw_circle(touchpad_data.x, touchpad_data.y, 10); } } }; + // NOTE: this example raises the BSP interrupt-task stack size via + // sdkconfig.defaults (CONFIG_SMARTPANLEE_SC01_PLUS_INTERRUPT_STACK_SIZE=8192); + // the touch controller is read from that task and its error-logging path + // needs more than the 4 KB BSP default. See the example README. if (!board.initialize_touch(touch_callback)) { logger.warn("Touch initialization did not complete cleanly"); } @@ -135,10 +143,18 @@ static void play_click(espp::SmartPanleeSc01Plus &board) { return; } + // Advance by however many bytes play_audio() actually queued (it enqueues + // only whole frames), not the requested size, so no samples are skipped. Stop + // as soon as the stream buffer is full rather than waiting for it to drain - + // this runs in the touch callback, so blocking would freeze the touch task + // for the whole click. The click comfortably fits in the stream buffer. size_t offset = 0; while (offset < audio_bytes.size()) { - auto bytes_to_play = std::min(audio_buffer_size, audio_bytes.size() - offset); - board.play_audio(audio_bytes.data() + offset, static_cast(bytes_to_play)); - offset += bytes_to_play; + auto chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); + size_t queued = board.play_audio(audio_bytes.data() + offset, static_cast(chunk)); + offset += queued; + if (queued < chunk) { + break; // stream buffer full for now; do not block the caller + } } } diff --git a/components/smartpanlee-sc01-plus/example/sdkconfig.defaults b/components/smartpanlee-sc01-plus/example/sdkconfig.defaults index 58f8b5d0f..3303bfb58 100755 --- a/components/smartpanlee-sc01-plus/example/sdkconfig.defaults +++ b/components/smartpanlee-sc01-plus/example/sdkconfig.defaults @@ -15,7 +15,10 @@ CONFIG_PARTITION_TABLE_CUSTOM=y CONFIG_PARTITION_TABLE_OFFSET=0x9000 CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" -CONFIG_SMARTPANLEE_SC01_PLUS_INTERRUPT_STACK_SIZE=4096 +# Raised from the 4 KB BSP default: the interrupt task runs the touch- +# controller I2C reads and, on a bus error, logs through espp's fmt +# color-print path, which needs more than 4 KB of stack. See the README. +CONFIG_SMARTPANLEE_SC01_PLUS_INTERRUPT_STACK_SIZE=8192 CONFIG_LV_DEF_REFR_PERIOD=16 CONFIG_LV_USE_THEME_DEFAULT=y diff --git a/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp b/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp index 0497d273f..98fd5f706 100755 --- a/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp +++ b/components/smartpanlee-sc01-plus/include/smartpanlee-sc01-plus.hpp @@ -247,11 +247,19 @@ class SmartPanleeSc01Plus : public BaseComponent { float volume() const; /// Queue raw PCM audio bytes for playback. /// \param data Audio payload to play. - void play_audio(std::span data); + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full); stream longer data + /// by calling repeatedly, advancing by the returned count + /// \note Must be called from task context, not from an ISR. + size_t play_audio(std::span data); /// Queue raw PCM audio bytes for playback. /// \param data Pointer to PCM audio bytes. /// \param num_bytes Number of bytes to play. - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full); stream longer + /// data by calling repeatedly, advancing by the returned count + /// \note Must be called from task context, not from an ISR. + size_t play_audio(const uint8_t *data, uint32_t num_bytes); /// Initialize and mount the optional microSD card with default settings. /// \return True if the card was mounted successfully, false otherwise. diff --git a/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp b/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp index ee88b5b53..92148b8c2 100755 --- a/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp +++ b/components/smartpanlee-sc01-plus/src/smartpanlee-sc01-plus.cpp @@ -381,15 +381,27 @@ void SmartPanleeSc01Plus::volume(float volume) { volume_ = std::clamp(volume, 0. float SmartPanleeSc01Plus::volume() const { return volume_; } -void SmartPanleeSc01Plus::play_audio(std::span data) { - play_audio(data.data(), static_cast(data.size())); +size_t SmartPanleeSc01Plus::play_audio(std::span data) { + return play_audio(data.data(), static_cast(data.size())); } -void SmartPanleeSc01Plus::play_audio(const uint8_t *data, uint32_t num_bytes) { +size_t SmartPanleeSc01Plus::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!audio_initialized_ || !audio_tx_stream_ || !data || num_bytes == 0) { - return; + return 0; + } + // Enqueue only whole 16-bit stereo frames (4 bytes) that actually fit: + // xStreamBufferSend can accept fewer bytes than requested when the buffer is + // nearly full, and a partial (non-frame-aligned) send would strand 1-3 bytes + // and corrupt the L/R framing of subsequent appends. Cap the request to the + // free space rounded down to a whole frame so the send is always + // frame-aligned; the number of bytes actually queued is returned so callers + // can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream_)); + sendable -= sendable % 4; + if (sendable == 0) { + return 0; } - xStreamBufferSend(audio_tx_stream_, data, num_bytes, 0); + return xStreamBufferSend(audio_tx_stream_, data, sendable, 0); } size_t SmartPanleeSc01Plus::rotated_display_width() const { diff --git a/components/t-deck/CMakeLists.txt b/components/t-deck/CMakeLists.txt index e0311d718..4265b1785 100644 --- a/components/t-deck/CMakeLists.txt +++ b/components/t-deck/CMakeLists.txt @@ -2,6 +2,6 @@ idf_component_register( INCLUDE_DIRS "include" SRC_DIRS "src" - REQUIRES driver esp_driver_i2s esp_driver_spi base_component display display_drivers fatfs i2c input_drivers interrupt gt911 spi task t_keyboard + REQUIRES driver esp_driver_i2s esp_driver_spi base_component codec display display_drivers fatfs i2c input_drivers interrupt gt911 spi task t_keyboard REQUIRED_IDF_TARGETS "esp32s3" ) diff --git a/components/t-deck/README.md b/components/t-deck/README.md index 09875a694..21245c7f3 100644 --- a/components/t-deck/README.md +++ b/components/t-deck/README.md @@ -15,6 +15,9 @@ subsystems. - GT911 capacitive touch input with LVGL integration helpers - T-Keyboard input and trackball pointer input - I2S speaker output with software volume / mute control +- Dual microphone recording through the ES7210 ADC (on its own I2S bus), + delivering 16-bit stereo frames to a callback with adjustable analog gain + via `microphone_volume()` - microSD mounting over SDSPI on the shared SPI host - Backlight brightness control and exposed peripheral helpers diff --git a/components/t-deck/example/README.md b/components/t-deck/example/README.md index 3c20445f6..ddd768edb 100644 --- a/components/t-deck/example/README.md +++ b/components/t-deck/example/README.md @@ -14,6 +14,24 @@ https://github.com/esp-cpp/espp/assets/213467/dc476c3d-dd9e-4b65-8c2d-9eda3ff3f3 ![image](https://github.com/esp-cpp/espp/assets/213467/4744d6ee-33bd-4907-8c58-3f3c2e5b7ba6) +## Configuration notes + +This example raises some espp BSP task stack sizes above their component +defaults via `sdkconfig.defaults`, because the example does more work in +those tasks than the defaults assume: + +- **Interrupt / touch task stack (`CONFIG_TDECK_INTERRUPT_STACK_SIZE` = 8192, up from the 4 KB + BSP default).** The interrupt task services the touch controller over + I2C; if an I2C transaction errors, the error is logged through espp's + `fmt`-based (colorized) logger, whose formatting path needs several KB of + stack. With only 4 KB this can overflow and corrupt memory. If you base + your own project on this example (or reproduce its functionality), keep + this override in your `sdkconfig` / `sdkconfig.defaults`. + +The example also enables `CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK`, which +turns any future task-stack overflow into an immediate, clearly-named panic +instead of silent heap corruption. + ## How to use example ### Hardware Required @@ -43,3 +61,17 @@ See the Getting Started Guide for full steps to configure and use ESP-IDF to bui - Trackball initialization and callback wiring - Optional uSD card mounting over SDSPI on the same SPI host - WAV playback through the onboard audio path +- Microphone recording and playback: the on-screen audio row (or the 'r' / + 'p' keys) records from the ES7210 into a PSRAM-preferred buffer and streams + it back through the mono speaker, with speaker / microphone volume buttons; + the measured effective capture rate is logged when a recording stops. + + Note on audio quality: the T-Deck's microphone is very low-sensitivity + (LilyGO's own firmware reads only ~200 counts for loud speech), and its + capture picks up random electrical impulse noise. The example therefore + post-processes each recording before playback: it de-glitches the impulse + noise (mark-and-interpolate), removes the DC offset, and applies an + RMS-normalized software makeup gain (the analog gain alone is not enough). + Only one microphone (MIC1) is used and mirrored to both channels for the + mono speaker. Even so, recordings are quiet and noisy - this is the limit + of the board's microphone hardware, not a configuration issue. diff --git a/components/t-deck/example/main/gui.cpp b/components/t-deck/example/main/gui.cpp index 7614e61bb..fc2602200 100644 --- a/components/t-deck/example/main/gui.cpp +++ b/components/t-deck/example/main/gui.cpp @@ -1,15 +1,14 @@ +#include +#include + #include "gui.hpp" void Gui::init_ui() { std::lock_guard lock(mutex_); - init_background(); - init_label(); - init_buttons(); + init_tabview(); + init_draw_tab(); + init_audio_tab(); init_circle_layer(); - // disable scrolling on the screen (so that it doesn't behave weirdly when - // rotated and drawing with your finger) - lv_obj_set_scrollbar_mode(lv_screen_active(), LV_SCROLLBAR_MODE_OFF); - lv_obj_clear_flag(lv_screen_active(), LV_OBJ_FLAG_SCROLLABLE); } void Gui::deinit_ui() { @@ -17,53 +16,134 @@ void Gui::deinit_ui() { lv_obj_clean(lv_screen_active()); } -void Gui::init_background() { - auto &tdeck = espp::TDeck::get(); - background_ = lv_obj_create(lv_screen_active()); - lv_obj_set_size(background_, tdeck.lcd_width(), tdeck.lcd_height()); - lv_obj_set_style_bg_color(background_, lv_color_make(0, 0, 0), 0); +void Gui::init_tabview() { + // the tabview gives each subsystem its own uncrowded page; the tab bar + // (top) is the page switcher + tabview_ = lv_tabview_create(lv_screen_active()); + lv_tabview_set_tab_bar_position(tabview_, LV_DIR_TOP); + lv_tabview_set_tab_bar_size(tabview_, TAB_BAR_HEIGHT); + lv_obj_set_size(tabview_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); + draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); + audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + // switching tabs is done with the tab buttons only: disable swipe + // scrolling of the content so drawing on the Draw tab cannot accidentally + // change pages + lv_obj_clear_flag(lv_tabview_get_content(tabview_), LV_OBJ_FLAG_SCROLLABLE); + // hide the touch-trail overlay whenever a non-drawing tab is shown + lv_obj_add_event_cb(tabview_, event_callback, LV_EVENT_VALUE_CHANGED, this); } -void Gui::init_label() { - label_ = lv_label_create(lv_screen_active()); +void Gui::init_draw_tab() { + // instructions on the left, with the rotate / clear buttons on the right + label_ = lv_label_create(draw_tab_); + lv_label_set_long_mode(label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(label_, lv_pct(70)); lv_label_set_text(label_, ""); - lv_obj_align(label_, LV_ALIGN_CENTER, 0, 0); - lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_CENTER, 0); -} + lv_obj_align(label_, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_set_style_text_align(label_, LV_TEXT_ALIGN_LEFT, 0); -void Gui::init_buttons() { - // a button in the top left which rotates the display through - // 0/90/180/270 degrees - rotate_button_ = lv_btn_create(lv_screen_active()); + rotate_button_ = lv_btn_create(draw_tab_); lv_obj_set_size(rotate_button_, 50, 50); - lv_obj_align(rotate_button_, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_align(rotate_button_, LV_ALIGN_TOP_RIGHT, 0, 0); lv_obj_t *rotate_label = lv_label_create(rotate_button_); lv_label_set_text(rotate_label, LV_SYMBOL_REFRESH); lv_obj_align(rotate_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(rotate_button_, event_callback, LV_EVENT_PRESSED, this); - // a button in the top right which clears the circles - clear_button_ = lv_btn_create(lv_screen_active()); + clear_button_ = lv_btn_create(draw_tab_); lv_obj_set_size(clear_button_, 50, 50); - lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 0); + lv_obj_align(clear_button_, LV_ALIGN_TOP_RIGHT, 0, 60); lv_obj_t *clear_label = lv_label_create(clear_button_); lv_label_set_text(clear_label, LV_SYMBOL_TRASH); lv_obj_align(clear_label, LV_ALIGN_CENTER, 0, 0); lv_obj_add_event_cb(clear_button_, event_callback, LV_EVENT_PRESSED, this); } -void Gui::init_circle_layer() { +void Gui::init_audio_tab() { + // a column: status line, volume line, then the buttons in a wrapping row + lv_obj_set_flex_flow(audio_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(audio_tab_, 10, 0); + + audio_status_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_status_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_status_label_, lv_pct(100)); + lv_label_set_text(audio_status_label_, "Idle"); + + audio_label_ = lv_label_create(audio_tab_); + lv_label_set_long_mode(audio_label_, LV_LABEL_LONG_WRAP); + lv_obj_set_width(audio_label_, lv_pct(100)); + update_audio_label(); + + lv_obj_t *row = lv_obj_create(audio_tab_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW_WRAP); + lv_obj_set_style_pad_column(row, 10, 0); + lv_obj_set_style_pad_row(row, 10, 0); + + struct ButtonSpec { + lv_obj_t **button; + const char *symbol; + }; + const ButtonSpec buttons[] = { + {&record_button_, LV_SYMBOL_AUDIO}, {&play_button_, LV_SYMBOL_PLAY}, + {&volume_down_button_, LV_SYMBOL_VOLUME_MID}, {&volume_up_button_, LV_SYMBOL_VOLUME_MAX}, + {&mic_down_button_, LV_SYMBOL_MINUS}, {&mic_up_button_, LV_SYMBOL_PLUS}, + }; + for (const auto &spec : buttons) { + *spec.button = lv_btn_create(row); + lv_obj_set_size(*spec.button, 44, 44); + lv_obj_t *label = lv_label_create(*spec.button); + lv_label_set_text(label, spec.symbol); + lv_obj_align(label, LV_ALIGN_CENTER, 0, 0); + lv_obj_add_event_cb(*spec.button, event_callback, LV_EVENT_PRESSED, this); + } + // remember the record / play button labels so set_record_active / + // set_play_active can swap their symbols + record_button_label_ = lv_obj_get_child(record_button_, 0); + play_button_label_ = lv_obj_get_child(play_button_, 0); + // color the volume buttons so the speaker and microphone pairs are + // distinguishable from each other + lv_obj_set_style_bg_color(mic_down_button_, lv_palette_main(LV_PALETTE_TEAL), 0); + lv_obj_set_style_bg_color(mic_up_button_, lv_palette_main(LV_PALETTE_TEAL), 0); +} + +void Gui::update_audio_label() { auto &tdeck = espp::TDeck::get(); + int speaker_volume = static_cast(tdeck.volume()); + int mic_volume = static_cast(tdeck.microphone_volume()); + // this is called every GUI tick; only reformat the label when a value + // actually changes (lv_label_set_text_fmt formats a new string each call) + if (speaker_volume == last_speaker_volume_ && mic_volume == last_mic_volume_) { + return; + } + last_speaker_volume_ = speaker_volume; + last_mic_volume_ = mic_volume; + // Pass the LVGL symbols as %s arguments rather than concatenating them into + // the format-string literal: cppcheck cannot expand the LVGL symbol macros + // and flags the literal concatenation as an unknown macro. + lv_label_set_text_fmt(audio_label_, "Speaker %d%% (%s/%s)\nMic %d%% (teal %s/%s)", speaker_volume, + LV_SYMBOL_VOLUME_MID, LV_SYMBOL_VOLUME_MAX, mic_volume, LV_SYMBOL_MINUS, + LV_SYMBOL_PLUS); +} + +void Gui::refresh_audio_label() { + std::lock_guard lock(mutex_); + update_audio_label(); +} + +void Gui::init_circle_layer() { + // a transparent, click-through overlay above the tabview which shows the + // touch trail (only populated while the Draw tab is active) circle_layer_ = lv_obj_create(lv_screen_active()); lv_obj_remove_style_all(circle_layer_); - lv_obj_set_size(circle_layer_, tdeck.lcd_width(), tdeck.lcd_height()); + lv_obj_set_size(circle_layer_, lv_display_get_horizontal_resolution(lv_display_get_default()), + lv_display_get_vertical_resolution(lv_display_get_default())); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_CLICKABLE); lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_SCROLLABLE); lv_obj_set_style_bg_opa(circle_layer_, LV_OPA_TRANSP, 0); - lv_obj_set_style_border_width(circle_layer_, 0, 0); - lv_obj_set_style_outline_width(circle_layer_, 0, 0); - lv_obj_set_style_shadow_width(circle_layer_, 0, 0); lv_obj_add_event_cb(circle_layer_, draw_circle_layer, LV_EVENT_DRAW_MAIN, this); lv_obj_move_foreground(circle_layer_); } @@ -72,6 +152,10 @@ bool Gui::update(std::mutex &m, std::condition_variable &cv) { { std::lock_guard lock(mutex_); lv_task_handler(); + // keep the audio volume label in sync with the live BSP state, so the + // first press of a volume button doesn't appear to jump from a stale + // default (the values are set by app_main after the Gui is constructed) + update_audio_label(); } std::unique_lock lock(m); cv.wait_for(lock, std::chrono::milliseconds(16)); @@ -87,13 +171,59 @@ void Gui::event_callback(lv_event_t *e) { case LV_EVENT_PRESSED: gui->on_pressed(e); break; + case LV_EVENT_VALUE_CHANGED: + gui->on_tab_changed(e); + break; default: break; } } +void Gui::on_tab_changed(lv_event_t *e) { + const auto *target = static_cast(lv_event_get_target(e)); + if (target != tabview_) { + return; + } + // the touch trail only belongs to the drawing tab; hide it (and its + // circles) everywhere else + if (lv_tabview_get_tab_active(tabview_) == 0) { + lv_obj_clear_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(circle_layer_, LV_OBJ_FLAG_HIDDEN); + } +} + void Gui::on_pressed(lv_event_t *e) { const auto *target = static_cast(lv_event_get_target(e)); + auto &tdeck = espp::TDeck::get(); + if (target == record_button_) { + logger_.info("Record button pressed"); + if (record_callback_) { + record_callback_(); + } + return; + } + if (target == play_button_) { + logger_.info("Play button pressed"); + if (play_callback_) { + play_callback_(); + } + return; + } + if (target == volume_down_button_ || target == volume_up_button_) { + float delta = target == volume_down_button_ ? -10.0f : 10.0f; + tdeck.volume(tdeck.volume() + delta); + logger_.info("Speaker volume: {:.0f}%", tdeck.volume()); + update_audio_label(); + return; + } + if (target == mic_down_button_ || target == mic_up_button_) { + float delta = target == mic_down_button_ ? -10.0f : 10.0f; + tdeck.microphone_volume(tdeck.microphone_volume() + delta); + logger_.info("Microphone volume: {:.0f}%", tdeck.microphone_volume()); + update_audio_label(); + return; + } if (target == rotate_button_) { logger_.info("Rotate button pressed"); next_rotation(); @@ -103,22 +233,48 @@ void Gui::on_pressed(lv_event_t *e) { } } +void Gui::set_record_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(record_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_AUDIO); + lv_obj_set_style_bg_color( + record_button_, active ? lv_palette_main(LV_PALETTE_RED) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_play_active(bool active) { + std::lock_guard lock(mutex_); + lv_label_set_text(play_button_label_, active ? LV_SYMBOL_STOP : LV_SYMBOL_PLAY); + lv_obj_set_style_bg_color( + play_button_, active ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_BLUE), + 0); +} + +void Gui::set_audio_status(std::string_view text) { + std::lock_guard lock(mutex_); + lv_label_set_text(audio_status_label_, std::string(text).c_str()); +} + void Gui::set_label_text(std::string_view text) { std::lock_guard lock(mutex_); lv_label_set_text(label_, std::string(text).c_str()); } +bool Gui::draw_page_active() { + std::lock_guard lock(mutex_); + return lv_tabview_get_tab_active(tabview_) == 0; +} + void Gui::next_rotation() { std::lock_guard lock(mutex_); - auto &tdeck = espp::TDeck::get(); clear_circles_impl(); auto rotation = lv_display_get_rotation(lv_display_get_default()); rotation = static_cast((static_cast(rotation) + 1) % 4); lv_display_set_rotation(lv_display_get_default(), rotation); // update the size of the screen-filling objects - lv_obj_set_size(background_, tdeck.rotated_display_width(), tdeck.rotated_display_height()); - lv_obj_align(label_, LV_ALIGN_CENTER, 0, 0); - lv_obj_set_size(circle_layer_, tdeck.rotated_display_width(), tdeck.rotated_display_height()); + int width = lv_display_get_horizontal_resolution(lv_display_get_default()); + int height = lv_display_get_vertical_resolution(lv_display_get_default()); + lv_obj_set_size(tabview_, width, height); + lv_obj_set_size(circle_layer_, width, height); lv_obj_align(circle_layer_, LV_ALIGN_CENTER, 0, 0); lv_obj_invalidate(circle_layer_); } diff --git a/components/t-deck/example/main/gui.hpp b/components/t-deck/example/main/gui.hpp index 78cf787f5..e9ee325ec 100644 --- a/components/t-deck/example/main/gui.hpp +++ b/components/t-deck/example/main/gui.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -23,13 +24,19 @@ /// dispatched through a single static trampoline (event_callback) into /// member functions, keeping all UI logic inside the class. /// -/// For this example the Gui shows: -/// * A label with instructions -/// * A button (top-left) which rotates the display through 0/90/180/270 -/// * A button (top-right) which clears the circles drawn on the screen -/// * A custom-drawn layer of circles which trail the most recent touches +/// The UI is organized as a tabview so each subsystem gets its own +/// uncrowded page: +/// * "Draw" tab: instructions, plus the rotate and clear buttons. Touching +/// the screen while this tab is active draws circles (on a transparent +/// overlay) and plays a click. +/// * "Audio" tab: record / play buttons (wired to the example via +/// callbacks) and speaker / microphone volume buttons (acting directly on +/// the BSP), with labels showing the audio state and current volumes. class Gui { public: + /// Callback invoked when the record / play buttons are pressed + using audio_button_callback_t = std::function; + /// Configuration for the Gui struct Config { espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Log verbosity @@ -48,10 +55,15 @@ class Gui { deinit_ui(); } - /// Set the text of the main label. Thread-safe. + /// Set the instruction text shown on the Draw tab. Thread-safe. /// @param text The text to display void set_label_text(std::string_view text); + /// Whether the Draw tab is currently active (used by the example to only + /// draw circles / play clicks for touches on that tab). Thread-safe. + /// @return True if the Draw tab is the active tab + bool draw_page_active(); + /// Draw a circle at the given screen coordinates, replacing the oldest /// circle if the maximum number are already visible. Thread-safe. /// @param x The x coordinate (screen space) @@ -66,8 +78,38 @@ class Gui { /// re-aligning the UI to match. Thread-safe. void next_rotation(); + /// Set the callback invoked when the record button is pressed + /// @param callback The callback to invoke + void set_record_callback(audio_button_callback_t callback) { + record_callback_ = std::move(callback); + } + + /// Set the callback invoked when the play button is pressed + /// @param callback The callback to invoke + void set_play_callback(audio_button_callback_t callback) { play_callback_ = std::move(callback); } + + /// Show whether a recording is in progress (turns the record button red + /// and changes its symbol to stop). Thread-safe. + /// @param active True while recording + void set_record_active(bool active); + + /// Show whether a playback is in progress (turns the play button green and + /// changes its symbol to stop). Thread-safe. + /// @param active True while playing + void set_play_active(bool active); + + /// Set the status line on the Audio tab (e.g. "Recording...", "Mic + /// unavailable"). Thread-safe. + /// @param text The text to display + void set_audio_status(std::string_view text); + + /// Refresh the audio volume label from the BSP's current volumes (e.g. + /// after the keyboard shortcuts change the volume). Thread-safe. + void refresh_audio_label(); + protected: static constexpr size_t MAX_CIRCLES = 100; + static constexpr int TAB_BAR_HEIGHT = 40; struct Circle { int x{0}; @@ -80,11 +122,15 @@ class Gui { void deinit_ui(); // the individual pieces of the UI, called from init_ui() - void init_background(); - void init_label(); - void init_buttons(); + void init_tabview(); + void init_draw_tab(); + void init_audio_tab(); void init_circle_layer(); + // update the audio volume label from the BSP's current volumes; called + // with the mutex held + void update_audio_label(); + // the LVGL update task: calls lv_task_handler() under the mutex bool update(std::mutex &m, std::condition_variable &cv); @@ -92,6 +138,7 @@ class Gui { // functions below based on the event target static void event_callback(lv_event_t *e); void on_pressed(lv_event_t *e); + void on_tab_changed(lv_event_t *e); // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); @@ -102,18 +149,39 @@ class Gui { void clear_circles_impl(); // LVGL objects - lv_obj_t *background_{nullptr}; + lv_obj_t *tabview_{nullptr}; + lv_obj_t *draw_tab_{nullptr}; + lv_obj_t *audio_tab_{nullptr}; lv_obj_t *label_{nullptr}; lv_obj_t *rotate_button_{nullptr}; lv_obj_t *clear_button_{nullptr}; + lv_obj_t *record_button_{nullptr}; + lv_obj_t *record_button_label_{nullptr}; + lv_obj_t *play_button_{nullptr}; + lv_obj_t *play_button_label_{nullptr}; + lv_obj_t *volume_down_button_{nullptr}; + lv_obj_t *volume_up_button_{nullptr}; + lv_obj_t *mic_down_button_{nullptr}; + lv_obj_t *mic_up_button_{nullptr}; + lv_obj_t *audio_label_{nullptr}; + // last values shown on audio_label_, so update_audio_label() (called every + // GUI tick) only reformats the text when a value actually changes + int last_speaker_volume_{-1}; + int last_mic_volume_{-1}; + lv_obj_t *audio_status_label_{nullptr}; lv_obj_t *circle_layer_{nullptr}; + audio_button_callback_t record_callback_{nullptr}; + audio_button_callback_t play_callback_{nullptr}; + std::array circles_; size_t next_circle_index_{0}; size_t visible_circle_count_{0}; espp::Task update_task_{{.callback = [this](auto &m, auto &cv) { return update(m, cv); }, - .task_config = {.name = "gui", .stack_size_bytes = 6 * 1024}}}; + // NOTE: rendering the tabview (nested containers + flex layout) uses + // noticeably more stack than a flat UI; 6 KB overflows + .task_config = {.name = "gui", .stack_size_bytes = 12 * 1024}}}; espp::Logger logger_; std::recursive_mutex mutex_; }; diff --git a/components/t-deck/example/main/t_deck_example.cpp b/components/t-deck/example/main/t_deck_example.cpp index c4881d947..d2aa6aa12 100644 --- a/components/t-deck/example/main/t_deck_example.cpp +++ b/components/t-deck/example/main/t_deck_example.cpp @@ -1,7 +1,15 @@ +#include +#include #include +#include +#include +#include #include #include +#include +#include + #include "t-deck.hpp" #include "gui.hpp" @@ -12,6 +20,38 @@ static std::vector audio_bytes; static bool load_audio(size_t &out_size, size_t &out_sample_rate); static void play_click(espp::TDeck &tdeck); +static void resample_click(uint32_t from_rate, uint32_t to_rate); + +// Run all audio at 16 kHz - the rate LilyGO's own T-Deck firmware uses for +// the ES7210 (the codec sounds clean there, and higher rates on this board +// produce a distorted / robotic capture). The speaker plays at the same rate +// so a recording plays back at the correct pitch with no runtime rate +// switching; the bundled 44.1 kHz click is resampled to 16 kHz at load. +static constexpr uint32_t AUDIO_SAMPLE_RATE_HZ = 16000; +static constexpr uint32_t MIC_SAMPLE_RATE_HZ = AUDIO_SAMPLE_RATE_HZ; + +// Audio recording state (written by the microphone callback, read/controlled +// from the GUI / keyboard callbacks and main loop). The recorded data is +// 16-bit interleaved stereo at the microphone sample rate. +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 2; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static std::atomic recording_start_us{0}; +static std::atomic recording_last_us{0}; +// number of bytes to discard at the start of a recording: the I2S RX DMA and +// the ES7210 emit a burst of settling garbage right after a recording begins, +// which otherwise plays back as a click of static at the front of the clip +static std::atomic warmup_bytes_remaining{0}; +// record / play toggles, shared by the on-screen buttons and the keyboard +// (assigned once the microphone is initialized) +static std::function toggle_record; +static std::function toggle_play; extern "C" void app_main(void) { espp::Logger logger({.tag = "T-Deck Example", .level = espp::Logger::Verbosity::INFO}); @@ -27,8 +67,11 @@ extern "C" void app_main(void) { if (!tdeck.initialize_sdcard(sdcard_config)) { logger.warn("Failed to initialize uSD card, there may not be a uSD card inserted!"); } - // initialize the sound - if (!tdeck.initialize_sound()) { + // initialize the sound at the example's fixed rate (see AUDIO_SAMPLE_RATE_HZ) + // so the speaker runs at that rate from the start - no runtime rate + // reconfiguration, which is where a speaker-vs-recording pitch mismatch + // could creep in + if (!tdeck.initialize_sound(AUDIO_SAMPLE_RATE_HZ)) { logger.error("Failed to initialize sound!"); return; } @@ -50,8 +93,10 @@ extern "C" void app_main(void) { // so the keyboard and touch callbacks below can call them directly. static Gui gui({.log_level = espp::Logger::Verbosity::INFO}); gui.set_label_text( - fmt::format("Touch the screen!\nPress the delete key or the {} button to clear " - "circles.\nPress the space key or the {} button to rotate the display.", + fmt::format("Touch the screen to draw!\nPress the delete key or the {} button to clear " + "circles.\nPress the space key or the {} button to rotate the display.\n" + "The Audio tab (or the 'r' / 'p' keys) records and plays back audio; " + "'n' / '$' / 'm' adjust / mute the speaker volume.", LV_SYMBOL_TRASH, LV_SYMBOL_REFRESH)); // initialize the Keyboard after the Gui exists so key presses can act on it @@ -76,11 +121,23 @@ extern "C" void app_main(void) { logger.info("Decreasing volume"); tdeck.volume(tdeck.volume() - 10.0f); logger.info("Volume: {}", tdeck.volume()); + gui.refresh_audio_label(); } else if (key == '$') { // '$' key will increase audio volume (right of 'm' key) logger.info("Increasing volume"); tdeck.volume(tdeck.volume() + 10.0f); logger.info("Volume: {}", tdeck.volume()); + gui.refresh_audio_label(); + } else if (key == 'r') { + // 'r' key toggles recording from the microphones + if (toggle_record) { + toggle_record(); + } + } else if (key == 'p') { + // 'p' key toggles playback of the recording + if (toggle_play) { + toggle_play(); + } } }; bool start_task = true; @@ -89,11 +146,17 @@ extern "C" void app_main(void) { return; } - // initialize the trackball + // initialize the trackball. This example uses the microphone, and per + // LilyGO's T-Deck documentation GPIO0 (the trackball's center / click + // button, shared with BOOT) is not available while the microphone is + // enabled - leaving it configured produces spurious interrupts that jitter + // the audio capture (a robotic / staticy recording). So initialize the + // trackball with the center button disabled; the four directional pins + // still work. auto trackball_callback = [&](const auto &trackball) { logger.debug("Trackball: {}", trackball); }; - if (!tdeck.initialize_trackball(trackball_callback)) { + if (!tdeck.initialize_trackball(trackball_callback, 10, /*enable_center_button=*/false)) { logger.error("Failed to initialize trackball!"); return; } @@ -110,13 +173,18 @@ extern "C" void app_main(void) { if (touchpad_data != previous_touchpad_data) { logger.info("Touch: {}", touchpad_data); previous_touchpad_data = touchpad_data; - // if there is a touch point, draw a circle and play a click sound - if (touchpad_data.num_touch_points > 0) { + // if there is a touch point on the Draw tab, draw a circle and play a + // click sound (touches on the other tabs go to their widgets) + if (touchpad_data.num_touch_points > 0 && gui.draw_page_active()) { play_click(tdeck); gui.draw_circle(touchpad_data.x, touchpad_data.y, 10); } } }; + // NOTE: this example raises the BSP interrupt-task stack size via + // sdkconfig.defaults (CONFIG_TDECK_INTERRUPT_STACK_SIZE=8192); the touch + // controller is read from that task and its error-logging path needs + // more than the 4 KB BSP default. See the example README. if (!tdeck.initialize_touch(touch_callback)) { logger.error("Failed to initialize touchpad!"); return; @@ -129,10 +197,13 @@ extern "C" void app_main(void) { logger.error("Failed to load audio file!"); return; } - logger.info("Loaded {} bytes of audio", wav_size); + logger.info("Loaded {} bytes of audio ({} Hz)", wav_size, wav_sample_rate); - logger.info("Setting audio sample rate to {} Hz", wav_sample_rate); - tdeck.audio_sample_rate(wav_sample_rate); + // Run the whole example at 16 kHz (see AUDIO_SAMPLE_RATE_HZ). The speaker + // was already initialized at that rate; the bundled click was decoded at + // its native rate, so resample it to 16 kHz once here so it plays at the + // right pitch through the 16 kHz speaker. + resample_click(static_cast(wav_sample_rate), AUDIO_SAMPLE_RATE_HZ); // unmute the audio and set the volume to 20% tdeck.mute(false); @@ -141,9 +212,267 @@ extern "C" void app_main(void) { // set the display brightness to be 75% tdeck.brightness(75.0f); - // now just loop forever + // Initialize the microphones (the ES7210 is on its own I2S bus, so pick + // the speaker's sample rate to make the recording directly playable) and + // buffer the recorded stereo frames; the recording auto-stops when the + // buffer is full and the main loop notices and updates the GUI + auto mic_callback = [](const uint8_t *data, size_t num_bytes) { + if (!recording) { + return; + } + // drop the warm-up (settling) bytes at the very start of the recording + size_t warm = warmup_bytes_remaining; + if (warm > 0) { + if (num_bytes <= warm) { + warmup_bytes_remaining = warm - num_bytes; + return; + } + data += warm; + num_bytes -= warm; + warmup_bytes_remaining = 0; + } + int64_t now = esp_timer_get_time(); + // stamp the true start of retained audio on the first kept sample so the + // measured capture rate excludes the warm-up period + if (recording_start_us == 0) { + recording_start_us = now; + } + size_t offset = recording_len; + size_t to_copy = std::min(num_bytes, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = now; + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + recording = false; + } + }; + // record at the ES7210's comfortable 16 kHz (see MIC_SAMPLE_RATE_HZ), which + // is independent of the speaker's rate (the ES7210 is on its own I2S bus) + bool have_mic = tdeck.initialize_microphone(mic_callback, MIC_SAMPLE_RATE_HZ); + if (have_mic) { + // The T-Deck's electret mics are genuinely low-sensitivity (LilyGO's own + // firmware reads only ~200 counts for loud speech next to the mic), so the + // analog stage is driven fairly hard here (~30 dB). Earlier this railed the + // ADC, but that was the mic's DC offset being amplified with the ES7210 + // high-pass filter disabled; the driver now enables the HPF, so the signal + // swings symmetrically around zero and this gain no longer saturates. The + // software auto-gain applied on stop (see the RMS normalization below) + // makes up whatever level remains; the mic +/- buttons adjust from here. + tdeck.microphone_volume(70.0f); + // allocate the recording buffer (16-bit interleaved stereo): prefer + // PSRAM, fall back to a couple of seconds in internal RAM + size_t bytes_per_second = tdeck.microphone_sample_rate() * 2 * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + logger.warn("Could not allocate a recording buffer; recording disabled"); + gui.set_audio_status("No recording buffer"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz stereo)", recording_capacity / 1024, + recording_capacity / bytes_per_second, tdeck.microphone_sample_rate()); + } + } else { + logger.warn("Could not initialize the microphone!"); + gui.set_audio_status("Mic unavailable (see log)"); + } + + // The record button / 'r' key toggles recording; the play button / 'p' key + // toggles playback of the recording (streamed to the speaker by the main + // loop) + toggle_record = [&]() { + if (!have_mic || recording_capacity == 0) { + logger.warn("Recording unavailable (no microphone / no buffer)"); + gui.set_audio_status("Mic unavailable (see log)"); + return; + } + if (recording) { + recording = false; // the main loop notices and logs the summary + } else { + playing = false; + gui.set_play_active(false); + recording_len = 0; + // discard the first ~250 ms of settling garbage; recording_start_us is + // stamped by the mic callback on the first retained sample + warmup_bytes_remaining = (tdeck.microphone_sample_rate() / 4) * 2 * sizeof(int16_t); + recording_start_us = 0; + recording_last_us = 0; + recording = true; + gui.set_record_active(true); + gui.set_audio_status("Recording..."); + } + }; + toggle_play = [&]() { + if (playing) { + playing = false; + gui.set_play_active(false); + gui.set_audio_status("Playback stopped"); + } else if (recording_len > 0) { + recording = false; + playing = true; + gui.set_play_active(true); + gui.set_audio_status("Playing..."); + } else { + logger.info("Nothing recorded yet; press the record button (or 'r') first"); + gui.set_audio_status("Nothing recorded yet"); + } + }; + gui.set_record_callback(toggle_record); + gui.set_play_callback(toggle_play); + + // Main loop: stream any active playback to the speaker and notice when a + // recording stops (either button press or the buffer filling up) + size_t play_offset = 0; + bool was_recording = false; while (true) { - std::this_thread::sleep_for(1s); + // feed the active playback in chunks, advancing by however much the + // speaker's stream buffer accepted + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_play_active(false); + gui.set_audio_status("Playback done"); + logger.info("Playback done"); + } else { + // Top up the speaker's stream buffer until it is full (play_audio + // returns less than requested once it can't accept more). A single + // chunk per loop leaves the buffer able to drain between iterations, + // which underruns and sounds like static / dropouts during a + // continuous playback. + while (play_offset < len) { + size_t chunk = std::min(len - play_offset, 16384); + size_t queued = tdeck.play_audio(recording_buffer + play_offset, chunk); + play_offset += queued; + if (queued < chunk) { + break; // stream buffer full for now + } + } + } + } else { + play_offset = 0; + } + // notice when the recording stopped (button press or buffer full) + bool now_recording = recording; + if (was_recording && !now_recording) { + gui.set_record_active(false); + gui.set_audio_status(fmt::format("Recorded {:.1f}s ({} plays)", + static_cast(recording_len) / + (tdeck.microphone_sample_rate() * 2 * sizeof(int16_t)), + LV_SYMBOL_PLAY)); + // report the measured capture rate: stereo frames recorded over the + // wall clock they took to arrive should match the nominal sample rate + size_t num_frames = recording_len / (2 * sizeof(int16_t)); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_frames / elapsed_s : 0.0f; + // Post-process the recording for playback on the T-Deck's MONO speaker. + // Use a single microphone (MIC1, the left slot - summing the two + // spatially separated mics comb-filters the sound) and mirror it to both + // stereo channels. The processing chain is: (1) de-glitch the electrical + // impulse noise, (2) remove the DC offset, (3) apply an RMS-normalized + // software makeup gain. The T-Deck mic is low-sensitivity (LilyGO's own + // firmware sees only ~200 counts for loud speech), so the makeup gain - + // not the analog stage - provides the loudness; RMS (rather than peak) + // sets it so a residual glitch cannot collapse the gain, and clipping + // catches any amplified outliers. + auto *samples = reinterpret_cast(recording_buffer); + // De-glitch: the ES7210 capture on this board carries random, bursty + // electrical impulse noise - roughly 0.3% of samples jump to |v| ~ 8000+ + // in runs of one to a few samples. (The de-spike frame gaps were measured + // to be non-periodic, i.e. line noise coupling into the mic / I2S rather + // than a framing or DMA-boundary artifact, so it cannot be removed by any + // codec register and has to be concealed here.) Detect each glitch by its + // large deviation from a local 7-point median (which tracks the real, + // slew-limited waveform, so legitimate speech is never flagged) and then + // linearly interpolate across each contiguous bad run from the good + // samples on either side. Interpolating a whole run conceals short bursts + // that a single-sample median replacement would leave behind. Done before + // the RMS/gain measurement so the glitches cannot inflate the makeup gain. + static constexpr int32_t kSpikeDelta = 2500; + size_t despiked = 0; + std::vector bad(num_frames, 0); + for (size_t i = 3; i + 3 < num_frames; i++) { + int32_t w[7]; + for (int k = 0; k < 7; k++) { + w[k] = samples[2 * (i - 3 + k)]; + } + for (int a = 1; a < 7; a++) { // insertion sort the 7-sample window + int32_t key = w[a]; + int b = a - 1; + while (b >= 0 && w[b] > key) { + w[b + 1] = w[b]; + b--; + } + w[b + 1] = key; + } + int32_t median = w[3]; + if (std::abs(static_cast(samples[2 * i]) - median) > kSpikeDelta) { + bad[i] = 1; + } + } + for (size_t i = 0; i < num_frames;) { + if (!bad[i]) { + i++; + continue; + } + size_t j = i; // [i, j) is a contiguous run of bad samples + while (j < num_frames && bad[j]) { + j++; + } + int32_t left = (i > 0) ? samples[2 * (i - 1)] : 0; + int32_t right = (j < num_frames) ? samples[2 * j] : left; + int32_t span = static_cast(j - i) + 1; + for (size_t k = i; k < j; k++) { + int32_t t = static_cast(k - i) + 1; + samples[2 * k] = static_cast(left + (right - left) * t / span); + } + despiked += j - i; + i = j; + } + int16_t min_l = 32767, max_l = -32768; + int64_t sum_l = 0, sum_sq = 0; + for (size_t i = 0; i < num_frames; i++) { + int16_t l = samples[2 * i]; // MIC1 (left) + min_l = std::min(min_l, l); + max_l = std::max(max_l, l); + sum_l += l; + } + int32_t dc_l = + num_frames ? static_cast(sum_l / static_cast(num_frames)) : 0; + for (size_t i = 0; i < num_frames; i++) { + int32_t v = samples[2 * i] - dc_l; + sum_sq += static_cast(v) * v; + } + double rms = num_frames ? std::sqrt(static_cast(sum_sq) / num_frames) : 0.0; + // target RMS ~4000 (about -18 dBFS) leaves headroom; cap the gain so a + // near-silent recording is not blown up into noise + static constexpr double target_rms = 4000.0; + double gain = rms > 1.0 ? std::clamp(target_rms / rms, 1.0, 64.0) : 1.0; + for (size_t i = 0; i < num_frames; i++) { + int32_t v = static_cast((samples[2 * i] - dc_l) * gain); + int16_t mono = static_cast(std::clamp(v, -32768, 32767)); + samples[2 * i] = mono; + samples[2 * i + 1] = mono; // mono: duplicate MIC1 to both channels + } + logger.info("Recorded {} frames in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal)", + num_frames, elapsed_s, effective_hz, tdeck.microphone_sample_rate()); + logger.info(" MIC1: raw min={} max={} dc={} rms={:.0f}; de-spiked {} samples; " + "applied software gain {:.1f}x", + min_l, max_l, dc_l, rms, despiked, gain); + } + was_recording = now_recording; + // tick faster while playing so the stream buffer is topped up well before + // it can drain (avoids underrun static); idle more slowly otherwise + std::this_thread::sleep_for(playing ? 10ms : 50ms); } //! [t-deck example] } @@ -177,13 +506,56 @@ static bool load_audio(size_t &out_size, size_t &out_sample_rate) { } static void play_click(espp::TDeck &tdeck) { - // use the tdeck.play_audio() function to play a sound, breaking it into - // audio_buffer_size chunks + // Enqueue the click without blocking the caller (this runs in the touch + // callback). play_audio() enqueues only whole frames and returns how many + // bytes it took, so advance by that count - advancing by the requested size + // would skip samples (at 16 kHz audio_buffer_size is not a multiple of a + // 4-byte frame, so this matters here). Stop as soon as the stream buffer is + // full rather than waiting for it to drain; blocking here would freeze the + // touch task for the whole click. The click comfortably fits in the stream + // buffer, so it plays in full in practice. auto audio_buffer_size = tdeck.audio_buffer_size(); size_t offset = 0; while (offset < audio_bytes.size()) { - size_t bytes_to_play = std::min(audio_buffer_size, audio_bytes.size() - offset); - tdeck.play_audio(audio_bytes.data() + offset, bytes_to_play); - offset += bytes_to_play; + size_t chunk = std::min(audio_buffer_size, audio_bytes.size() - offset); + size_t queued = tdeck.play_audio(audio_bytes.data() + offset, chunk); + offset += queued; + if (queued < chunk) { + break; // stream buffer full for now; do not block the caller + } + } +} + +// Resample the loaded click (16-bit interleaved stereo) in place from +// from_rate to to_rate with linear interpolation, so it plays at the correct +// pitch through the speaker running at to_rate. +static void resample_click(uint32_t from_rate, uint32_t to_rate) { + if (from_rate == to_rate || audio_bytes.empty() || to_rate == 0) { + return; + } + size_t in_frames = audio_bytes.size() / (2 * sizeof(int16_t)); + if (in_frames < 2) { + return; + } + // Copy the samples into a properly-aligned int16_t buffer: audio_bytes is a + // byte vector, so reinterpreting its storage as int16_t* and dereferencing it + // would be unaligned / undefined behavior. + std::vector in(in_frames * 2); + std::memcpy(in.data(), audio_bytes.data(), in.size() * sizeof(int16_t)); + size_t out_frames = static_cast(static_cast(in_frames) * to_rate / from_rate); + std::vector out(out_frames * 2); + double step = static_cast(from_rate) / to_rate; + for (size_t j = 0; j < out_frames; ++j) { + double src = j * step; + size_t i0 = static_cast(src); + size_t i1 = std::min(i0 + 1, in_frames - 1); + double frac = src - i0; + for (int ch = 0; ch < 2; ++ch) { + double a = in[2 * i0 + ch]; + double b = in[2 * i1 + ch]; + out[2 * j + ch] = static_cast(a + (b - a) * frac); + } } + audio_bytes.resize(out.size() * sizeof(int16_t)); + std::memcpy(audio_bytes.data(), out.data(), audio_bytes.size()); } diff --git a/components/t-deck/example/sdkconfig.defaults b/components/t-deck/example/sdkconfig.defaults index 100486ab5..996de4a5a 100644 --- a/components/t-deck/example/sdkconfig.defaults +++ b/components/t-deck/example/sdkconfig.defaults @@ -42,3 +42,17 @@ CONFIG_LV_USE_THEME_DEFAULT=y CONFIG_LV_THEME_DEFAULT_DARK=y CONFIG_LV_THEME_DEFAULT_GROW=y CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=30 + +# PSRAM (octal PSRAM on the WROOM-1 R8 / R16V modules); used for the audio +# recording buffer in the example +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y + +# catch task stack overflows immediately (debug watchpoint on the stack end) +# instead of letting them silently corrupt the heap +CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y +# The interrupt task (touch I2C) logs errors through the fmt color-print +# path, which needs more than the 4 KB BSP default; give it 8 KB here so a +# transient touch-bus error cannot overflow its stack. See the README. +CONFIG_TDECK_INTERRUPT_STACK_SIZE=8192 diff --git a/components/t-deck/include/t-deck.hpp b/components/t-deck/include/t-deck.hpp index 7d8a0e257..25999741b 100644 --- a/components/t-deck/include/t-deck.hpp +++ b/components/t-deck/include/t-deck.hpp @@ -21,6 +21,7 @@ #include #include "base_component.hpp" +#include "es7210.hpp" #include "gt911.hpp" #include "i2c.hpp" #include "interrupt.hpp" @@ -194,8 +195,15 @@ class TDeck : public BaseComponent { /// \see trackball_data() /// \see trackball_read() /// \see set_trackball_sensitivity() + /// \param enable_center_button Whether to configure the trackball's center + /// (click) button on GPIO0. Set this to false when the microphone is + /// in use: per LilyGO's documentation GPIO0 is not available while the + /// microphone is enabled, and leaving the button interrupt configured + /// on GPIO0 while recording produces a burst of spurious interrupts + /// that jitters the real-time audio capture. The four directional + /// quadrature pins are unaffected and still work. bool initialize_trackball(const trackball_callback_t &trackball_cb = nullptr, - int sensitivity = 10); + int sensitivity = 10, bool enable_center_button = true); /// Get the trackball /// \return A shared pointer to the trackball @@ -456,13 +464,67 @@ class TDeck : public BaseComponent { float volume() const; /// Play audio - /// \param data The audio data to play - void play_audio(const std::vector &data); + /// \param data The audio data to play (16-bit signed interleaved stereo) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. + size_t play_audio(const std::vector &data); /// Play audio - /// \param data The audio data to play + /// \param data The audio data to play (16-bit signed interleaved stereo) /// \param num_bytes The number of bytes to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) + /// \note This function is non-blocking and queues the data for the audio + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + /// \note Must be called from task context, not from an ISR. + size_t play_audio(const uint8_t *data, uint32_t num_bytes); + + ///////////////////////////////////////////////////////////////////////////// + // Microphone + ///////////////////////////////////////////////////////////////////////////// + + /// Alias for the microphone callback, called with recorded audio data + using microphone_callback_t = std::function; + + /// Initialize the microphones (the dual microphone array through the + /// ES7210 ADC, on its own I2S bus) and start delivering audio data to the + /// provided callback + /// \param callback The callback to call with recorded audio data: 16-bit + /// signed interleaved stereo. The two populated ES7210 microphones are + /// delivered as MIC1 (TDM slot 0) on the left channel and MIC3 (TDM + /// slot 2) on the right, at \p sample_rate + /// \param sample_rate The sample rate for the microphones, in Hz. The + /// ES7210 is on a separate I2S bus from the speaker amplifier, so + /// this is independent of the speaker's sample rate. + /// \param task_config The configuration for the microphone task + /// \return true if the microphone was successfully initialized, false + /// otherwise + /// \note The callback runs in the microphone task's context, so the task's + /// stack must be large enough for whatever the callback does with + /// the audio data + bool initialize_microphone(const microphone_callback_t &callback, uint32_t sample_rate = 16000, + const espp::Task::BaseConfig &task_config = {.name = "microphone", + .stack_size_bytes = 4096, + .priority = 10, + .core_id = 1}); + + /// Get the microphone sample rate + /// \return The microphone sample rate, in Hz + uint32_t microphone_sample_rate() const; + + /// Set the microphone volume + /// \param volume The volume as a percentage (0 - 100), mapped onto the + /// ES7210 analog microphone gain range (0 dB - +37.5 dB) + void microphone_volume(float volume); + + /// Get the microphone volume + /// \return The microphone volume as a percentage (0 - 100) + float microphone_volume() const; protected: TDeck(); @@ -472,6 +534,7 @@ class TDeck : public BaseComponent { void on_trackball_interrupt(const espp::Interrupt::Event &event); bool initialize_i2s(uint32_t default_audio_rate); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); // common: // internal i2c (touchscreen, keyboard) @@ -495,6 +558,7 @@ class TDeck : public BaseComponent { // // static constexpr gpio_num_t es7210_int_io = GPIO_NUM_17; static constexpr gpio_num_t dmic_clk_io = GPIO_NUM_17; + static constexpr auto mic_i2s_port = I2S_NUM_1; // Audio Out (MAX98357A) static constexpr auto i2s_port = I2S_NUM_0; @@ -670,5 +734,16 @@ class TDeck : public BaseComponent { std::vector audio_tx_buffer; StreamBufferHandle_t audio_tx_stream; i2s_std_config_t audio_std_cfg; + + // microphone (ES7210 on its own I2S bus) + std::shared_ptr> es7210_i2c_device_; + std::atomic microphone_initialized_{false}; + microphone_callback_t microphone_callback_{nullptr}; + std::unique_ptr microphone_task_{nullptr}; + i2s_chan_handle_t audio_rx_handle{nullptr}; + std::vector audio_rx_buffer; + std::atomic mic_sample_rate_{0}; + // microphone volume (percent), mapped onto the ES7210 analog gain range + std::atomic mic_volume_{70.0f}; }; // class TDeck } // namespace espp diff --git a/components/t-deck/src/audio.cpp b/components/t-deck/src/audio.cpp index cea2507fa..22dace6df 100644 --- a/components/t-deck/src/audio.cpp +++ b/components/t-deck/src/audio.cpp @@ -1,3 +1,10 @@ +#include +#include +#include +#include + +#include + #include "t-deck.hpp" using namespace espp; @@ -6,6 +13,41 @@ using namespace espp; // Audio Functions // //////////////////////// +// Map a 0-100% microphone volume onto the ES7210's analog gain steps +// (GAIN_0DB .. GAIN_37_5DB) +static es7210_gain_value_t microphone_gain_from_volume(float volume) { + int step = static_cast(std::lround(volume / 100.0f * static_cast(GAIN_37_5DB))); + step = std::clamp(step, static_cast(GAIN_0DB), static_cast(GAIN_37_5DB)); + return static_cast(step); +} + +// Map a PCM sample rate onto the codec's audio_hal sample-rate enum. The ES7210 +// coefficient table is selected by this enum, so it must match the rate the I2S +// clocks actually generate; an unsupported rate falls back to 16 kHz (the rate +// this board records at). +static audio_hal_iface_samples_t audio_hal_samples_from_rate(uint32_t rate) { + switch (rate) { + case 8000: + return AUDIO_HAL_08K_SAMPLES; + case 11025: + return AUDIO_HAL_11K_SAMPLES; + case 16000: + return AUDIO_HAL_16K_SAMPLES; + case 22050: + return AUDIO_HAL_22K_SAMPLES; + case 24000: + return AUDIO_HAL_24K_SAMPLES; + case 32000: + return AUDIO_HAL_32K_SAMPLES; + case 44100: + return AUDIO_HAL_44K_SAMPLES; + case 48000: + return AUDIO_HAL_48K_SAMPLES; + default: + return AUDIO_HAL_16K_SAMPLES; + } +} + bool TDeck::initialize_i2s(uint32_t default_audio_rate) { logger_.info("initializing i2s driver"); logger_.debug("Using newer I2S standard"); @@ -91,9 +133,9 @@ bool TDeck::initialize_sound(uint32_t default_audio_rate, bool TDeck::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { // Queue the next I2S out frame to write - uint16_t available = xStreamBufferBytesAvailable(audio_tx_stream); - int buffer_size = audio_tx_buffer.size(); - available = std::min(available, buffer_size); + size_t available = xStreamBufferBytesAvailable(audio_tx_stream); + size_t buffer_size = audio_tx_buffer.size(); + available = std::min(available, buffer_size); uint8_t *buffer = &audio_tx_buffer[0]; memset(buffer, 0, buffer_size); @@ -110,7 +152,7 @@ bool TDeck::audio_task_callback(std::mutex &m, std::condition_variable &cv, bool // NOTE: we are actually using int16_t samples, so we need to adjust the // pointers to the buffer to the correct positions. int16_t *ptr = reinterpret_cast(buffer); - for (int i = 0; i < (available / 2); ++i) { + for (size_t i = 0; i < (available / 2); ++i) { int32_t sample = (ptr[i] * volume_) / 100.0f; ptr[i] = static_cast(std::clamp(sample, INT16_MIN, INT16_MAX)); } @@ -147,9 +189,248 @@ void TDeck::audio_sample_rate(uint32_t sample_rate) { i2s_channel_enable(audio_tx_handle); } -void TDeck::play_audio(const std::vector &data) { play_audio(data.data(), data.size()); } +size_t TDeck::play_audio(const std::vector &data) { + return play_audio(data.data(), data.size()); +} -void TDeck::play_audio(const uint8_t *data, uint32_t num_bytes) { - // don't block here - xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); +size_t TDeck::play_audio(const uint8_t *data, uint32_t num_bytes) { + // guard against being called before initialize_sound() (audio_tx_stream is + // not valid until then) and against empty input + if (!sound_initialized_ || !data || num_bytes == 0) { + return 0; + } + // Enqueue only whole 16-bit stereo frames (4 bytes) that actually fit: + // xStreamBufferSend can accept fewer bytes than requested when the buffer is + // nearly full, and a partial (non-frame-aligned) send would strand 1-3 bytes + // and corrupt the L/R framing of subsequent appends. Cap the request to the + // free space rounded down to a whole frame so the send is always + // frame-aligned. This runs in task context, so the non-ISR send with a 0 + // timeout never blocks. The number of bytes actually queued is returned so + // callers can stream data larger than the buffer. + size_t sendable = std::min(num_bytes, xStreamBufferSpacesAvailable(audio_tx_stream)); + sendable -= sendable % 4; + if (sendable == 0) { + return 0; + } + return xStreamBufferSend(audio_tx_stream, data, sendable, 0); } + +////////////////////////// +// Microphone Functions // +////////////////////////// + +bool TDeck::initialize_microphone(const microphone_callback_t &callback, uint32_t sample_rate, + const espp::Task::BaseConfig &task_config) { + logger_.info("Initializing microphone"); + if (microphone_initialized_) { + logger_.warn("Microphone already initialized, not initializing again!"); + return false; + } + if (!callback) { + logger_.error("A callback is required to receive the recorded audio data"); + return false; + } + microphone_callback_ = callback; + + // The ES7210 has its own I2S bus (separate from the speaker amplifier), so + // make a receive-only master channel with its own clocks; the codec is a + // slave clocked from the MCLK / SCLK / LRCK we generate. + // + // The ES7210 outputs its four ADC channels as a 4-slot TDM frame (MIC1 -> + // slot 0, MIC2 -> slot 1, MIC3 -> slot 2, MIC4 -> slot 3), so read all four + // slots. The microphone task extracts the populated microphones (the T-Deck + // wires MIC1 and MIC3, i.e. slots 0 and 2 - one from each ADC pair) and + // presents them as 16-bit stereo. A 2-slot standard-I2S read would only see + // the first ADC pair (MIC1 + the unpopulated MIC2). + i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(mic_i2s_port, I2S_ROLE_MASTER); + ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, nullptr, &audio_rx_handle)); + i2s_tdm_config_t mic_tdm_cfg = { + .clk_cfg = {.sample_rate_hz = sample_rate, + .clk_src = I2S_CLK_SRC_DEFAULT, + .ext_clk_freq_hz = 0, + .mclk_multiple = I2S_MCLK_MULTIPLE_256, + // Let the driver derive BCLK from the slot count/width. The + // ES7210 coefficient table is programmed for MCLK = 256 x + // sample_rate (see MCLK_DIV_FRE in es7210.cpp), which + // mclk_multiple pins here. Forcing bclk_div (e.g. 8) makes + // the driver raise MCLK to satisfy the divider, so the codec + // receives an MCLK its coefficients were not computed for; + // its ADC decimation then drifts against LRCK and slips a + // frame every so often, which reads as sparse full-scale + // (0x8000) spikes on top of otherwise-clean audio. + .bclk_div = 0}, + .slot_cfg = {.data_bit_width = I2S_DATA_BIT_WIDTH_16BIT, + .slot_bit_width = I2S_SLOT_BIT_WIDTH_16BIT, + .slot_mode = I2S_SLOT_MODE_STEREO, + .slot_mask = static_cast(I2S_TDM_SLOT0 | I2S_TDM_SLOT1 | + I2S_TDM_SLOT2 | I2S_TDM_SLOT3), + .ws_width = I2S_TDM_AUTO_WS_WIDTH, + .ws_pol = false, + .bit_shift = true, + .left_align = false, + .big_endian = false, + .bit_order_lsb = false, + .skip_mask = false, + .total_slot = I2S_TDM_AUTO_SLOT_NUM}, + .gpio_cfg = {.mclk = es7210_mclk_io, + .bclk = es7210_sclk_io, + .ws = es7210_lrck_io, + .dout = GPIO_NUM_NC, + .din = es7210_sdout_io, + .invert_flags = {.mclk_inv = false, .bclk_inv = false, .ws_inv = false}}, + }; + ESP_ERROR_CHECK(i2s_channel_init_tdm_mode(audio_rx_handle, &mic_tdm_cfg)); + mic_sample_rate_ = sample_rate; + // the read buffer holds the raw 4-slot TDM frames (twice the size of the + // 2-channel stereo the callback receives) + audio_rx_buffer.resize(2 * calc_audio_buffer_size(sample_rate)); + ESP_ERROR_CHECK(i2s_channel_enable(audio_rx_handle)); + + // Configure the ES7210 ADC now that its clocks are running. The ES7210 always + // emits a fixed 4-slot TDM frame; this board only populates MIC1 (slot 0) and + // MIC3 (slot 2), which the callback compacts to the left / right channels of + // the delivered 16-bit stereo. + std::error_code ec; + es7210_i2c_device_ = internal_i2c_.add_device( + { + .device_address = 0x40, + .timeout_ms = static_cast(internal_i2c_.config().timeout_ms), + .scl_speed_hz = internal_i2c_.config().clk_speed, + .log_level = espp::Logger::Verbosity::WARN, + }, + ec); + if (!es7210_i2c_device_) { + logger_.error("Could not initialize ES7210 I2C device: {}", ec.message()); + i2s_channel_disable(audio_rx_handle); + i2s_del_channel(audio_rx_handle); + audio_rx_handle = nullptr; + return false; + } + set_es7210_write(espp::make_i2c_addressed_write(es7210_i2c_device_)); + set_es7210_read(espp::make_i2c_addressed_read_register(es7210_i2c_device_)); + + // Enable all four ADC channels so the ES7210 always emits a fixed 4-slot + // TDM frame ([MIC1, MIC2, MIC3, MIC4]). Enabling only a subset can make the + // codec pack fewer slots, which shifts the frame boundary against the fixed + // 4-slot read below and periodically lands the extraction on a floating, + // railing slot (heard as a robotic / static buzz on top of the audio). The + // T-Deck only populates MIC1 and MIC3, so the task keeps slots 0 and 2 and + // discards the unpopulated MIC2 / MIC4 slots. + es7210_mic_select(static_cast(ES7210_INPUT_MIC1 | ES7210_INPUT_MIC2 | + ES7210_INPUT_MIC3 | ES7210_INPUT_MIC4)); + + audio_hal_codec_config_t es7210_cfg{}; + es7210_cfg.codec_mode = AUDIO_HAL_CODEC_MODE_ENCODE; + es7210_cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; + es7210_cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; + es7210_cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; + // configure the ES7210 for the requested capture rate (rather than assuming + // 16 kHz) so its coefficient table matches the I2S clocks + es7210_cfg.i2s_iface.samples = audio_hal_samples_from_rate(sample_rate); + if (es7210_adc_init(&es7210_cfg) != ESP_OK) { + logger_.error("Could not initialize the ES7210 codec"); + i2s_channel_disable(audio_rx_handle); + i2s_del_channel(audio_rx_handle); + audio_rx_handle = nullptr; + return false; + } + if (es7210_adc_config_i2s(AUDIO_HAL_CODEC_MODE_ENCODE, &es7210_cfg.i2s_iface) != ESP_OK) { + logger_.error("ES7210 I2S config failed"); + } + // NOTE: do not write register 0x08 (master/slave & channels) here. The + // es7210 driver leaves it at its reset default in slave mode, which is + // correct for this board. Forcing 0x08 = 0x20 (as some MicroPython T-Deck + // ports do for their async PWM MCLK) reconfigures the channel/decimation + // path on our hardware-MCLK setup into a half-rate, every-other-frame-zero + // capture (MIC1 samples drop out and the glitch/de-spike rate rises), so it + // is deliberately left alone. + // Enable the ES7210 digital high-pass filter on the ADC channels to strip + // the microphone DC offset. The espp es7210 driver leaves these registers at + // their reset value (HPF off), so the (large, positive) DC offset from the + // T-Deck's electret mics is amplified by the analog gain and rails the ADC: + // captures come out all-positive with a big constant bias (e.g. MIC1 min=0 + // max=24672) that reads as a quiet-but-distorted / robotic recording, and at + // higher gain saturates to full scale. Removing the DC here lets the signal + // swing symmetrically around zero so the analog gain can be raised safely. + // These are the driver's documented "quick setup" HPF values, and are what + // the (working) esp-box microphone path uses. The T-Deck populates MIC1 and + // MIC3, one from each ADC pair, so both pairs' HPFs are enabled. + // write_register clears the error code on success, so a later successful + // write would mask an earlier failure; remember the first failure explicitly. + std::error_code hpf_ec, first_hpf_ec; + auto write_hpf = [&](uint8_t reg, uint8_t val) { + es7210_i2c_device_->write_register(reg, std::vector{val}, hpf_ec); + if (hpf_ec && !first_hpf_ec) { + first_hpf_ec = hpf_ec; + } + }; + write_hpf(0x22, 0x0a); // ADC1/2 HPF1 + write_hpf(0x23, 0x2a); // ADC1/2 HPF2 + write_hpf(0x20, 0x0a); // ADC3/4 HPF2 + write_hpf(0x21, 0x2a); // ADC3/4 HPF1 + if (first_hpf_ec) { + logger_.warn("Could not enable the ES7210 high-pass filter: {}", first_hpf_ec.message()); + } + // apply the stored microphone volume (the driver's init leaves the analog + // gain at its 0 dB minimum, which records very quietly) + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + es7210_adc_ctrl_state(AUDIO_HAL_CODEC_MODE_ENCODE, AUDIO_HAL_CTRL_START); + + using namespace std::placeholders; + microphone_task_ = espp::Task::make_unique({ + .callback = std::bind(&TDeck::microphone_task_callback, this, _1, _2, _3), + .task_config = task_config, + }); + + if (!microphone_task_->start()) { + i2s_channel_disable(audio_rx_handle); + i2s_del_channel(audio_rx_handle); + audio_rx_handle = nullptr; + microphone_task_.reset(); + return false; + } + + microphone_initialized_ = true; + return true; +} + +uint32_t TDeck::microphone_sample_rate() const { return mic_sample_rate_; } + +bool TDeck::microphone_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + size_t bytes_read = 0; + // Use a finite read timeout (not portMAX_DELAY) so this task returns + // periodically and can observe a stop request; an infinite read would block + // Task::stop() from joining during teardown. + auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), + &bytes_read, pdMS_TO_TICKS(100)); + if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { + // audio_rx_buffer holds 4-slot TDM frames: [MIC1, MIC2, MIC3, MIC4]. + // Compact the populated microphones (MIC1 -> slot 0, MIC3 -> slot 2) down + // to 16-bit stereo in place (the output region is the front half, so this + // is safe). + auto *samples = reinterpret_cast(audio_rx_buffer.data()); + size_t num_frames = bytes_read / (4 * sizeof(int16_t)); + for (size_t i = 0; i < num_frames; i++) { + samples[2 * i] = samples[4 * i]; // MIC1 -> left + samples[2 * i + 1] = samples[4 * i + 2]; // MIC3 -> right + } + microphone_callback_(audio_rx_buffer.data(), num_frames * 2 * sizeof(int16_t)); + } + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running +} + +void TDeck::microphone_volume(float volume) { + mic_volume_ = std::clamp(volume, 0.0f, 100.0f); + if (microphone_initialized_) { + es7210_adc_set_gain_all(microphone_gain_from_volume(mic_volume_)); + } +} + +float TDeck::microphone_volume() const { return mic_volume_; } diff --git a/components/t-deck/src/t-deck.cpp b/components/t-deck/src/t-deck.cpp index cf80dc9fb..b1e230a66 100644 --- a/components/t-deck/src/t-deck.cpp +++ b/components/t-deck/src/t-deck.cpp @@ -88,7 +88,8 @@ std::shared_ptr TDeck::keyboard() const { return keyboard_; } // Trackball Functions // ///////////////////////// -bool TDeck::initialize_trackball(const TDeck::trackball_callback_t &trackball_cb, int sensitivity) { +bool TDeck::initialize_trackball(const TDeck::trackball_callback_t &trackball_cb, int sensitivity, + bool enable_center_button) { if (pointer_input_) { logger_.warn("Trackball already initialized, not initializing again!"); return false; @@ -102,12 +103,20 @@ bool TDeck::initialize_trackball(const TDeck::trackball_callback_t &trackball_cb // store the callback trackball_callback_ = trackball_cb; - // add the interrupts for the trackball + // add the interrupts for the trackball directional (quadrature) pins interrupts_.add_interrupt(trackball_up_interrupt_pin); interrupts_.add_interrupt(trackball_down_interrupt_pin); interrupts_.add_interrupt(trackball_left_interrupt_pin); interrupts_.add_interrupt(trackball_right_interrupt_pin); - interrupts_.add_interrupt(trackball_btn_interrupt_pin); + // the center (click) button is on GPIO0, which LilyGO documents as + // unavailable while the microphone is enabled; skip it when requested so + // microphone recording is not disrupted by spurious GPIO0 interrupts + if (enable_center_button) { + interrupts_.add_interrupt(trackball_btn_interrupt_pin); + } else { + logger_.info("Skipping trackball center button (GPIO0) - unavailable while the microphone " + "is in use"); + } // set the sensitivity set_trackball_sensitivity(sensitivity);