From 943467b0769e16fdf7199b0d3dc68be109957698 Mon Sep 17 00:00:00 2001 From: Aruna Tennakoon Date: Tue, 15 Sep 2026 20:23:28 +0700 Subject: [PATCH 1/3] feat: camera device with WebRTC live view Adds a camera device that answers the SinricPro portal and app WebRTC signaling, and an example that streams JPEG frames over a DataChannel. - sinricpro_camera: getCameraCapabilities reports webrtc and webrtcAudio; getWebRTCAnswer decodes the base64 offer, flattens iceServers[].urls and returns a base64 answer. The component gains no new dependencies. - examples/camera: a webrtc_camera component built on esp_peer and esp32-camera, with resolution, frame rate, flash, flip and mirror controls, automatic quality, and the XIAO ESP32S3 Sense microphone. Ten board profiles are selectable in menuconfig. - sinricpro_set_response_message() lets a callback tell the client why a request failed. - Kconfig SINRICPRO_MAX_MESSAGE_SIZE (default 16 KB). Fixes a core bug: a server message larger than the websocket client's 2 KB buffer is delivered as several data events, and each piece was parsed as a complete message, so the message was lost. Pieces are now reassembled, and ping, pong and close frames are no longer handed to the JSON parser. Tested: host tests for signing, reassembly and camera signaling; the switch example rebuilt; the camera example built for esp32 and esp32s3 on IDF v6.1; live view on an AI-Thinker ESP32-CAM through the SinricPro portal. --- .github/workflows/build-test.yml | 28 + CHANGELOG.md | 23 + CMakeLists.txt | 3 + Kconfig | 10 + README.md | 5 + docs/api-reference.md | 45 ++ examples/camera/CMakeLists.txt | 9 + examples/camera/README.md | 82 ++ .../components/webrtc_camera/CMakeLists.txt | 24 + .../webrtc_camera/camera_controls.c | 309 ++++++++ .../webrtc_camera/camera_controls.h | 67 ++ .../webrtc_camera/idf_component.yml | 10 + .../webrtc_camera/include/webrtc_camera.h | 126 +++ .../components/webrtc_camera/jpeg_streamer.c | 103 +++ .../components/webrtc_camera/jpeg_streamer.h | 64 ++ .../components/webrtc_camera/webrtc_camera.c | 718 ++++++++++++++++++ .../webrtc_camera/webrtc_camera_priv.h | 21 + examples/camera/main/CMakeLists.txt | 2 + examples/camera/main/Kconfig.projbuild | 58 ++ examples/camera/main/camera_boards.c | 191 +++++ examples/camera/main/camera_boards.h | 22 + examples/camera/main/camera_example.c | 369 +++++++++ examples/camera/main/idf_component.yml | 8 + examples/camera/partitions.csv | 4 + examples/camera/sdkconfig.defaults | 20 + examples/camera/sdkconfig.defaults.esp32s3 | 4 + include/sinricpro.h | 17 + include/sinricpro_camera.h | 143 ++++ src/capabilities/camera_controller.c | 233 ++++++ src/capabilities/camera_controller.h | 83 ++ src/core/sinricpro_core.c | 20 +- src/core/sinricpro_frame_assembler.c | 93 +++ src/core/sinricpro_frame_assembler.h | 74 ++ src/core/sinricpro_websocket.c | 56 +- src/devices/sinricpro_camera.c | 190 +++++ test/host/run.sh | 42 +- test/host/shims/esp_event.h | 9 + test/host/shims/mbedtls/base64.h | 8 +- test/host/shims/shims.c | 68 ++ test/host/test_camera_controller.c | 226 ++++++ test/host/test_frame_assembler.c | 177 +++++ 41 files changed, 3733 insertions(+), 31 deletions(-) create mode 100644 examples/camera/CMakeLists.txt create mode 100644 examples/camera/README.md create mode 100644 examples/camera/components/webrtc_camera/CMakeLists.txt create mode 100644 examples/camera/components/webrtc_camera/camera_controls.c create mode 100644 examples/camera/components/webrtc_camera/camera_controls.h create mode 100644 examples/camera/components/webrtc_camera/idf_component.yml create mode 100644 examples/camera/components/webrtc_camera/include/webrtc_camera.h create mode 100644 examples/camera/components/webrtc_camera/jpeg_streamer.c create mode 100644 examples/camera/components/webrtc_camera/jpeg_streamer.h create mode 100644 examples/camera/components/webrtc_camera/webrtc_camera.c create mode 100644 examples/camera/components/webrtc_camera/webrtc_camera_priv.h create mode 100644 examples/camera/main/CMakeLists.txt create mode 100644 examples/camera/main/Kconfig.projbuild create mode 100644 examples/camera/main/camera_boards.c create mode 100644 examples/camera/main/camera_boards.h create mode 100644 examples/camera/main/camera_example.c create mode 100644 examples/camera/main/idf_component.yml create mode 100644 examples/camera/partitions.csv create mode 100644 examples/camera/sdkconfig.defaults create mode 100644 examples/camera/sdkconfig.defaults.esp32s3 create mode 100644 include/sinricpro_camera.h create mode 100644 src/capabilities/camera_controller.c create mode 100644 src/capabilities/camera_controller.h create mode 100644 src/core/sinricpro_frame_assembler.c create mode 100644 src/core/sinricpro_frame_assembler.h create mode 100644 src/devices/sinricpro_camera.c create mode 100644 test/host/shims/esp_event.h create mode 100644 test/host/test_camera_controller.c create mode 100644 test/host/test_frame_assembler.c diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 7d4cf84..e4d5986 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -57,6 +57,34 @@ jobs: . $IDF_PATH/export.sh idf.py size + # Camera live view. The example's board profiles cover ESP32 and ESP32-S3 only, + # so it cannot join the main matrix's S2 and C3 targets. + build-camera: + runs-on: ubuntu-latest + container: + image: espressif/idf:v5.1 + strategy: + fail-fast: false + matrix: + idf-target: [esp32, esp32s3] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: Clean managed components + run: | + find . -type d -name "managed_components" -exec rm -rf {} + || true + + - name: Build camera for ${{ matrix.idf-target }} + working-directory: examples/camera + run: | + . $IDF_PATH/export.sh + idf.py set-target ${{ matrix.idf-target }} + idf.py build + # Local control without the mDNS announcement. Keeps the UDP listener but # drops the mdns component, which is the configuration for a board whose app # partition cannot take it. A gate that is never built is a gate that breaks. diff --git a/CHANGELOG.md b/CHANGELOG.md index b5550ed..abd1ce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [Unreleased] + +### Features + +- feat: camera device with WebRTC live view in the SinricPro portal and app. + `sinricpro_camera_on_webrtc_offer()` answers `getWebRTCAnswer` using the ICE + servers the server sends, and `getCameraCapabilities` reports `webrtc` and + `webrtcAudio` so viewers know what the firmware supports. +- feat: `examples/camera` streams JPEG frames over a WebRTC DataChannel through a + `webrtc_camera` component built on `esp_peer`, with resolution, frame rate, + flash, flip and mirror controls, automatic quality, and the XIAO ESP32S3 Sense + microphone. The SinricPro component itself gains no dependencies. +- feat: `sinricpro_set_response_message()`, so a callback can tell the client why + a request failed. +- feat: Kconfig `SINRICPRO_MAX_MESSAGE_SIZE` (default 16 KB). + +### Fixes + +- fix: a server message larger than the websocket client's 2 KB buffer is posted + as several data events, and each piece was parsed as a complete message, so the + message was lost. The pieces are now reassembled. +- fix: ping, pong and close frames are no longer handed to the JSON parser. + ## [1.3.0] ### Features diff --git a/CMakeLists.txt b/CMakeLists.txt index a1fbab0..838645f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ idf_component_register( "src/core/sinricpro_event_limiter.c" "src/core/sinricpro_udp.c" "src/core/sinricpro_mdns.c" + "src/core/sinricpro_frame_assembler.c" "src/devices/sinricpro_switch.c" "src/devices/sinricpro_motion_sensor.c" "src/devices/sinricpro_contact_sensor.c" @@ -39,6 +40,8 @@ idf_component_register( "src/devices/sinricpro_windowac.c" "src/devices/sinricpro_tv.c" "src/devices/sinricpro_speaker.c" + "src/devices/sinricpro_camera.c" + "src/capabilities/camera_controller.c" "src/capabilities/power_state_controller.c" "src/capabilities/setting_controller.c" "src/capabilities/push_notification.c" diff --git a/Kconfig b/Kconfig index 91fdc9d..881ee7a 100644 --- a/Kconfig +++ b/Kconfig @@ -69,6 +69,16 @@ menu "SinricPro Configuration" Maximum number of messages that can be queued for sending. + config SINRICPRO_MAX_MESSAGE_SIZE + int "Maximum incoming message size (bytes)" + default 16384 + range 2048 65536 + help + Largest server message the SDK reassembles. The websocket client + delivers frames in 2 KB pieces; a message above this limit is + dropped whole rather than parsed in fragments. A camera WebRTC + offer carries SDP plus TURN credentials and needs several KB. + config SINRICPRO_AUTO_RECONNECT bool "Enable auto-reconnection" default y diff --git a/README.md b/README.md index c5f45c0..3432448 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Official ESP-IDF component for [SinricPro](https://sinric.pro) - Control your ES - ✅ **Voice Control** - Works with Alexa and Google Home - ✅ **Real-time** - WebSocket-based bidirectional communication - ✅ **Local Control** - Answers the app over the LAN when the cloud is down +- ✅ **Camera Live View** - WebRTC streaming to the SinricPro portal and app - ✅ **Secure** - HMAC-SHA256 message signatures - ✅ **Reliable** - Auto-reconnection and heartbeat monitoring - ✅ **Event-driven** - ESP event loop integration @@ -49,6 +50,9 @@ All devices below have complete API support and working examples: - ✅ **TV** - Volume, mute, media control, input selection, channels - ✅ **Speaker** - Volume, mute, media control, equalizer, modes +### Cameras +- ✅ **Camera** - WebRTC live view in the SinricPro portal and app, with remote resolution, frame rate and flash control + ### Additional Devices (API Only) - ✅ Air Quality Sensor - PM1, PM2.5, PM10 measurements - ✅ Power Sensor - Voltage, current, power monitoring @@ -125,6 +129,7 @@ The component includes **13 complete working examples** demonstrating all device | [Blinds](examples/blinds/) | Motorized blinds/curtains | ⭐⭐⭐ Complex | DC motor, L298N | | [TV](examples/tv/) | Media control & channels | ⭐⭐⭐ Complex | Simulated/IR | | [Speaker](examples/speaker/) | Audio control & equalizer | ⭐⭐⭐ Complex | Simulated | +| [Camera](examples/camera/) | WebRTC live view in the portal and app | ⭐⭐⭐ Complex | ESP32/ESP32-S3 camera with PSRAM | Each example includes: - Complete working code diff --git a/docs/api-reference.md b/docs/api-reference.md index 1bf7017..a7df116 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -13,6 +13,10 @@ esp_err_t sinricpro_deinit(void); bool sinricpro_is_connected(void); uint32_t sinricpro_get_timestamp(void); const char* sinricpro_get_version(void); + +/* From inside a device callback: replace the response's "OK" / + * "Device did not handle request" text with a reason the client can show. */ +esp_err_t sinricpro_set_response_message(const char *message); ``` #### Configuration Structure @@ -87,6 +91,46 @@ esp_err_t sinricpro_switch_send_notification( ); ``` +### Camera Device API + +Live view in the SinricPro portal and app over WebRTC. The component handles +signaling; the peer connection and streaming live in the `webrtc_camera` +component of [examples/camera](../examples/camera/). + +```c +sinricpro_device_handle_t sinricpro_camera_create(const char *device_id); +esp_err_t sinricpro_camera_delete(sinricpro_device_handle_t device); + +esp_err_t sinricpro_camera_on_power_state(sinricpro_device_handle_t device, + sinricpro_camera_power_state_callback_t callback, + void *user_data); + +/* One entry per URL; strings are valid only during the callback. */ +typedef struct { + const char *url; /* "stun:…", "turn:…?transport=udp", "turns:…:443?transport=tcp" */ + const char *username; /* "" when none */ + const char *credential; /* "" when none */ +} sinricpro_ice_server_t; + +/* Set *answer_sdp to a malloc()ed SDP answer containing every local candidate; + * the SDK frees it. May block while ICE gathers (up to ~5 s). */ +typedef bool (*sinricpro_camera_webrtc_offer_callback_t)( + const char *device_id, const char *offer_sdp, + const sinricpro_ice_server_t *ice_servers, size_t ice_server_count, + char **answer_sdp, void *user_data); + +/* Registering the callback makes getCameraCapabilities report webrtc: true. */ +esp_err_t sinricpro_camera_on_webrtc_offer(sinricpro_device_handle_t device, + sinricpro_camera_webrtc_offer_callback_t callback, + void *user_data); + +/* Reported as webrtcAudio, so viewers request an audio track. */ +esp_err_t sinricpro_camera_enable_webrtc_audio(sinricpro_device_handle_t device, bool enabled); + +esp_err_t sinricpro_camera_send_power_state_event(sinricpro_device_handle_t device, + bool state, const char *cause); +``` + ### Event System ```c @@ -122,6 +166,7 @@ Access via `idf.py menuconfig` → `Component config` → `SinricPro Configurati - **Enable Debug Logging** - Verbose logging for troubleshooting - **Event Queue Size** - Maximum queued events - **Message Queue Size** - Maximum queued messages +- **Maximum Incoming Message Size** - Largest reassembled server message (default 16 KB; camera offers need several KB) - **Auto-reconnection** - Enable/disable auto-reconnection - **Reconnection Interval** - Time between reconnection attempts - **Max Devices** - Maximum number of registered devices diff --git a/examples/camera/CMakeLists.txt b/examples/camera/CMakeLists.txt new file mode 100644 index 0000000..731033c --- /dev/null +++ b/examples/camera/CMakeLists.txt @@ -0,0 +1,9 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.16) + +# Add the parent components directory so we can find sinricpro component +set(EXTRA_COMPONENT_DIRS "${CMAKE_CURRENT_LIST_DIR}/../..") # Use local component + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(camera_example) diff --git a/examples/camera/README.md b/examples/camera/README.md new file mode 100644 index 0000000..df5672f --- /dev/null +++ b/examples/camera/README.md @@ -0,0 +1,82 @@ +# Camera Example (WebRTC live view) + +Streams an ESP32 or ESP32-S3 camera to the SinricPro portal and app, from anywhere. Video is JPEG over an encrypted WebRTC DataChannel; signaling runs through the SinricPro connection, and STUN/TURN servers arrive with each viewer's offer, so viewing works outside your LAN. Viewers can change resolution and frame rate and toggle flash, flip and mirror, and quality drops automatically on slow links. XIAO ESP32S3 Sense also streams its onboard microphone. + +## Requirements + +- ESP-IDF 5.1 or later +- An ESP32 or ESP32-S3 camera board **with PSRAM** +- 4 MB of flash or more +- A Wi-Fi signal of **−75 dBm or better** at the board. Below about −80 dBm the Wi-Fi driver's transmit buffers stop recycling fast enough and the DTLS handshake cannot complete, even though free heap looks healthy. + +## Portal setup + +1. Create a device of type **Camera**. +2. In **Camera Stream Configuration**, set Board to **ESP32** and Streaming Protocol to **WebRTC**. +3. Copy the device ID, app key and app secret. + +## Configure and build + +Edit the credentials at the top of [main/camera_example.c](main/camera_example.c): + +```c +#define WIFI_SSID "WIFI_SSID" +#define WIFI_PASS "WIFI_PASS" +#define DEVICE_ID "DEVICE_ID" +#define APP_KEY "APP_KEY" +#define APP_SECRET "APP_SECRET" +``` + +Select the target, then your board under **SinricPro Camera Example → Camera board**: + +```bash +cd examples/camera +idf.py set-target esp32 # or esp32s3 +idf.py menuconfig +idf.py build flash monitor +``` + +| Board | Target | +| --- | --- | +| AI-Thinker ESP32-CAM (default on esp32) | esp32 | +| ESP-EYE, M5Camera A/B, ESP-WROVER-KIT, LILYGO T-Camera | esp32 | +| XIAO ESP32S3 Sense (default on esp32s3) | esp32s3 | +| Freenove ESP32-S3, ESP32-S3 WROOM (PWDN 38), GOOUUU ESP32-S3 | esp32s3 | + +`sdkconfig.defaults.esp32s3` assumes octal PSRAM, as on the XIAO Sense and Freenove N8R8. For a board with quad PSRAM, set `CONFIG_SPIRAM_MODE_QUAD` instead. + +Open **Preview** on the camera in the portal, or tap the camera in the app. + +## How it works + +| Piece | Where | +| --- | --- | +| `getCameraCapabilities`, `getWebRTCAnswer` | SinricPro component: `sinricpro_camera_on_webrtc_offer()` | +| Peer connection, JPEG streaming, viewer controls | [components/webrtc_camera](components/webrtc_camera/), built on Espressif's `esp_peer` | +| Pin mappings | [main/camera_boards.c](main/camera_boards.c) | + +The SinricPro component itself does not depend on `esp_peer` or `esp32-camera`; the `webrtc_camera` component can be copied into your own project. + +Signaling is a single offer/answer exchange without trickle ICE, so the session gathers every local candidate before answering, and the answer callback blocks for up to about 5 seconds. One viewer is served at a time; a new offer replaces the current viewer. + +## Memory settings + +[sdkconfig.defaults](sdkconfig.defaults) moves Wi-Fi, lwIP and mbedTLS allocations into PSRAM. On classic ESP32 that is what leaves enough contiguous internal RAM for the Wi-Fi driver's transmit buffers once the TLS connection to SinricPro is open. The example also keeps the DataChannel caches small on classic ESP32 for the same reason. + +`CONFIG_FREERTOS_HZ=1000` matters too: the session sends one DataChannel fragment per tick, so at the default 100 Hz throughput is capped at about 100 kB/s. + +## Troubleshooting + +| Symptom | Check | +| --- | --- | +| Viewer shows one frozen frame, device logs `streaming: yes` | Wi-Fi signal; the log prints RSSI at connect and every 30 s | +| `dtlsState=connecting` in `chrome://webrtc-internals` | Wi-Fi signal and free internal heap in the 30-second log line | +| Viewer reports "firmware does not support live view" | The WebRTC offer callback is not registered | +| `Camera init failed` | Board selection in menuconfig, ribbon cable, and PSRAM mode | +| `PSRAM is not available` | `CONFIG_SPIRAM`, and octal versus quad PSRAM on ESP32-S3 | + +## Limits + +- Portal and app only. Alexa and Google Home need a native H.264 video track. +- WebRTC signaling needs the cloud connection: local control's UDP transport cannot carry an offer. +- Snapshot and motion upload are not implemented yet. diff --git a/examples/camera/components/webrtc_camera/CMakeLists.txt b/examples/camera/components/webrtc_camera/CMakeLists.txt new file mode 100644 index 0000000..aba21ca --- /dev/null +++ b/examples/camera/components/webrtc_camera/CMakeLists.txt @@ -0,0 +1,24 @@ +# driver/gpio.h moved out of the monolithic driver component in ESP-IDF 5.3; +# from 6.0 that component no longer provides it at all. +if("${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}" VERSION_GREATER_EQUAL "5.3") + set(gpio_component esp_driver_gpio) +else() + set(gpio_component driver) +endif() + +idf_component_register( + SRCS + "webrtc_camera.c" + "jpeg_streamer.c" + "camera_controls.c" + INCLUDE_DIRS + "include" + REQUIRES + esp_peer + esp32-camera + PRIV_REQUIRES + cjson + esp_timer + esp_wifi + ${gpio_component} +) diff --git a/examples/camera/components/webrtc_camera/camera_controls.c b/examples/camera/components/webrtc_camera/camera_controls.c new file mode 100644 index 0000000..d531aa5 --- /dev/null +++ b/examples/camera/components/webrtc_camera/camera_controls.c @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#include "camera_controls.h" +#include +#include "cJSON.h" +#include "driver/gpio.h" + +typedef struct { + framesize_t size; + const char *name; +} resolution_name_t; + +static const resolution_name_t RESOLUTIONS[] = { + {FRAMESIZE_QVGA, "QVGA"}, {FRAMESIZE_CIF, "CIF"}, {FRAMESIZE_VGA, "VGA"}, {FRAMESIZE_SVGA, "SVGA"}, + {FRAMESIZE_XGA, "XGA"}, {FRAMESIZE_HD, "HD"}, {FRAMESIZE_SXGA, "SXGA"}, {FRAMESIZE_UXGA, "UXGA"}, +}; +#define RESOLUTION_COUNT (sizeof(RESOLUTIONS) / sizeof(RESOLUTIONS[0])) + +typedef struct { + uint16_t interval_percent; + uint8_t quality_offset; +} quality_level_t; + +/* Frame rate drops first because it keeps detail; JPEG compression follows when rate alone is not enough. */ +static const quality_level_t LEVELS[] = {{100, 0}, {150, 0}, {200, 6}, {300, 12}, {400, 18}, {600, 24}}; +#define LEVEL_COUNT (sizeof(LEVELS) / sizeof(LEVELS[0])) +#define DEGRADE_AFTER_FRAMES 2 +#define RECOVER_AFTER_FRAMES 20 +#define MAX_JPEG_QUALITY 63 /* esp32-camera: a higher number compresses harder */ + +static int clamp_fps(int fps) +{ + if (fps < CAMERA_CONTROLS_MIN_FPS) { + return CAMERA_CONTROLS_MIN_FPS; + } + return fps > CAMERA_CONTROLS_MAX_FPS ? CAMERA_CONTROLS_MAX_FPS : fps; +} + +static const char *resolution_name(framesize_t size) +{ + for (size_t i = 0; i < RESOLUTION_COUNT; i++) { + if (RESOLUTIONS[i].size == size) { + return RESOLUTIONS[i].name; + } + } + return ""; +} + +static void set_level(camera_controls_t *controls, uint8_t level) +{ + controls->congested_frames = 0; + controls->good_frames = 0; + if (level == controls->level) { + return; + } + controls->level = level; + + sensor_t *sensor = esp_camera_sensor_get(); + if (sensor != NULL) { + int quality = controls->base_quality + LEVELS[level].quality_offset; + sensor->set_quality(sensor, quality > MAX_JPEG_QUALITY ? MAX_JPEG_QUALITY : quality); + } + controls->state_changed = true; +} + +static bool set_resolution(camera_controls_t *controls, const char *name) +{ + sensor_t *sensor = esp_camera_sensor_get(); + for (size_t i = 0; i < RESOLUTION_COUNT; i++) { + if (strcmp(name, RESOLUTIONS[i].name) != 0 || RESOLUTIONS[i].size > controls->max_frame_size) { + continue; + } + if (RESOLUTIONS[i].size == controls->frame_size || sensor == NULL || + sensor->set_framesize(sensor, RESOLUTIONS[i].size) != 0) { + return false; + } + controls->frame_size = RESOLUTIONS[i].size; + set_level(controls, 0); + return true; + } + return false; +} + +static bool set_fps(camera_controls_t *controls, int fps) +{ + fps = clamp_fps(fps); + if (fps == controls->fps) { + return false; + } + controls->fps = fps; + set_level(controls, 0); + return true; +} + +static bool set_flash(camera_controls_t *controls, bool on) +{ + if (controls->flash_gpio < 0 || on == controls->flash) { + return false; + } + gpio_set_level((gpio_num_t)controls->flash_gpio, on ? 1 : 0); + controls->flash = on; + controls->state_changed = true; + return true; +} + +static bool set_flip(camera_controls_t *controls, bool on) +{ + sensor_t *sensor = esp_camera_sensor_get(); + if (on == controls->flip || sensor == NULL || sensor->set_vflip(sensor, on) != 0) { + return false; + } + controls->flip = on; + return true; +} + +static bool set_mirror(camera_controls_t *controls, bool on) +{ + sensor_t *sensor = esp_camera_sensor_get(); + if (on == controls->mirror || sensor == NULL || sensor->set_hmirror(sensor, on) != 0) { + return false; + } + controls->mirror = on; + return true; +} + +static bool set_auto_quality(camera_controls_t *controls, bool on) +{ + if (on == controls->auto_quality) { + return false; + } + controls->auto_quality = on; + if (!on) { + set_level(controls, 0); + } + return true; +} + +void camera_controls_init(camera_controls_t *controls, framesize_t max_frame_size, int flash_gpio, + uint32_t frame_interval_ms, bool auto_quality) +{ + memset(controls, 0, sizeof(*controls)); + controls->frame_size = FRAMESIZE_VGA; + controls->base_quality = 12; + controls->auto_quality = auto_quality; + controls->flash_gpio = flash_gpio; + + sensor_t *sensor = esp_camera_sensor_get(); + if (sensor != NULL) { + controls->frame_size = sensor->status.framesize; + controls->base_quality = sensor->status.quality; + controls->flip = sensor->status.vflip; + controls->mirror = sensor->status.hmirror; + } + controls->max_frame_size = max_frame_size == FRAMESIZE_INVALID ? controls->frame_size : max_frame_size; + + if (flash_gpio >= 0) { + gpio_reset_pin((gpio_num_t)flash_gpio); + gpio_set_direction((gpio_num_t)flash_gpio, GPIO_MODE_OUTPUT); + gpio_set_level((gpio_num_t)flash_gpio, 0); + } + + controls->fps = clamp_fps((int)(1000 / (frame_interval_ms > 0 ? frame_interval_ms : 1))); +} + +bool camera_controls_apply(camera_controls_t *controls, const char *message, size_t length) +{ + cJSON *json = cJSON_ParseWithLength(message, length); + if (json == NULL) { + return false; + } + + bool changed = false; + cJSON *type = cJSON_GetObjectItem(json, "type"); + if (cJSON_IsString(type) && strcmp(type->valuestring, "set") == 0) { + cJSON *item = cJSON_GetObjectItem(json, "resolution"); + if (cJSON_IsString(item)) { + changed |= set_resolution(controls, item->valuestring); + } + item = cJSON_GetObjectItem(json, "fps"); + if (cJSON_IsNumber(item)) { + changed |= set_fps(controls, item->valueint); + } + item = cJSON_GetObjectItem(json, "flash"); + if (cJSON_IsBool(item)) { + changed |= set_flash(controls, cJSON_IsTrue(item)); + } + item = cJSON_GetObjectItem(json, "flip"); + if (cJSON_IsBool(item)) { + changed |= set_flip(controls, cJSON_IsTrue(item)); + } + item = cJSON_GetObjectItem(json, "mirror"); + if (cJSON_IsBool(item)) { + changed |= set_mirror(controls, cJSON_IsTrue(item)); + } + item = cJSON_GetObjectItem(json, "autoQuality"); + if (cJSON_IsBool(item)) { + changed |= set_auto_quality(controls, cJSON_IsTrue(item)); + } + + /* Report state even when a value was rejected, so the viewer resyncs its controls. */ + controls->state_changed = true; + } + + cJSON_Delete(json); + return changed; +} + +void camera_controls_on_frame_result(camera_controls_t *controls, bool completed, uint32_t duration_ms) +{ + if (!controls->auto_quality) { + return; + } + + uint32_t interval = camera_controls_frame_interval_ms(controls); + /* A frame that takes longer to send than the frame interval means the link cannot keep up. */ + if (!completed || duration_ms > interval) { + controls->good_frames = 0; + if (++controls->congested_frames >= DEGRADE_AFTER_FRAMES && controls->level + 1 < LEVEL_COUNT) { + set_level(controls, controls->level + 1); + } + } else if (duration_ms * 2 < interval) { + controls->congested_frames = 0; + if (++controls->good_frames >= RECOVER_AFTER_FRAMES && controls->level > 0) { + set_level(controls, controls->level - 1); + } + } else { + controls->congested_frames = 0; + controls->good_frames = 0; + } +} + +void camera_controls_viewer_left(camera_controls_t *controls) +{ + set_flash(controls, false); + if (controls->level > 0) { + set_level(controls, 0); + } + controls->state_changed = false; +} + +uint32_t camera_controls_frame_interval_ms(const camera_controls_t *controls) +{ + return (1000 / (uint32_t)controls->fps) * LEVELS[controls->level].interval_percent / 100; +} + +static char *print_and_delete(cJSON *json) +{ + char *text = json != NULL ? cJSON_PrintUnformatted(json) : NULL; + cJSON_Delete(json); + return text; +} + +char *camera_controls_capabilities_json(const camera_controls_t *controls) +{ + cJSON *json = cJSON_CreateObject(); + if (json == NULL) { + return NULL; + } + + cJSON_AddStringToObject(json, "type", "capabilities"); + cJSON *resolutions = cJSON_AddArrayToObject(json, "resolutions"); + for (size_t i = 0; i < RESOLUTION_COUNT && resolutions != NULL; i++) { + if (RESOLUTIONS[i].size <= controls->max_frame_size) { + cJSON_AddItemToArray(resolutions, cJSON_CreateString(RESOLUTIONS[i].name)); + } + } + cJSON_AddNumberToObject(json, "minFps", CAMERA_CONTROLS_MIN_FPS); + cJSON_AddNumberToObject(json, "maxFps", CAMERA_CONTROLS_MAX_FPS); + cJSON_AddBoolToObject(json, "flash", controls->flash_gpio >= 0); + cJSON_AddBoolToObject(json, "flip", true); + cJSON_AddBoolToObject(json, "mirror", true); + return print_and_delete(json); +} + +char *camera_controls_state_json(const camera_controls_t *controls) +{ + cJSON *json = cJSON_CreateObject(); + if (json == NULL) { + return NULL; + } + + uint32_t interval = camera_controls_frame_interval_ms(controls); + /* One decimal, matching what the viewers display. */ + double effective_fps = (double)(uint32_t)(10000.0 / (interval > 0 ? interval : 1) + 0.5) / 10.0; + + cJSON_AddStringToObject(json, "type", "state"); + cJSON_AddStringToObject(json, "resolution", resolution_name(controls->frame_size)); + cJSON_AddNumberToObject(json, "fps", controls->fps); + cJSON_AddBoolToObject(json, "flash", controls->flash); + cJSON_AddBoolToObject(json, "flip", controls->flip); + cJSON_AddBoolToObject(json, "mirror", controls->mirror); + cJSON_AddBoolToObject(json, "autoQuality", controls->auto_quality); + cJSON_AddNumberToObject(json, "qualityLevel", controls->level); + cJSON_AddNumberToObject(json, "effectiveFps", effective_fps); + return print_and_delete(json); +} + +bool camera_controls_take_state_changed(camera_controls_t *controls) +{ + bool changed = controls->state_changed; + controls->state_changed = false; + return changed; +} diff --git a/examples/camera/components/webrtc_camera/camera_controls.h b/examples/camera/components/webrtc_camera/camera_controls.h new file mode 100644 index 0000000..38978b6 --- /dev/null +++ b/examples/camera/components/webrtc_camera/camera_controls.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#ifndef CAMERA_CONTROLS_H +#define CAMERA_CONTROLS_H + +#include +#include +#include +#include "esp_camera.h" + +/* Camera settings a viewer can change over the DataChannel, plus automatic quality adaptation. + * Control messages are UTF-8 JSON text; binary messages carry JPEG fragments. + * device -> viewer {"type":"capabilities","resolutions":["QVGA","VGA","SVGA"],"minFps":1,"maxFps":15, + * "flash":true,"flip":true,"mirror":true} + * device -> viewer {"type":"state","resolution":"VGA","fps":5,"flash":false,"flip":false,"mirror":false, + * "autoQuality":true,"qualityLevel":0,"effectiveFps":5.0} + * viewer -> device {"type":"set", ...any of resolution, fps, flash, flip, mirror, autoQuality} + * Not thread-safe: use it from the task that owns the peer. */ + +#define CAMERA_CONTROLS_MIN_FPS 1 +#define CAMERA_CONTROLS_MAX_FPS 15 + +typedef struct { + framesize_t frame_size; + framesize_t max_frame_size; + int flash_gpio; + int fps; + int base_quality; + bool flash; + bool flip; + bool mirror; + bool auto_quality; + uint8_t level; + uint8_t congested_frames; + uint8_t good_frames; + bool state_changed; +} camera_controls_t; + +/** Call after esp_camera_init(): the current sensor settings become the starting state. */ +void camera_controls_init(camera_controls_t *controls, framesize_t max_frame_size, int flash_gpio, + uint32_t frame_interval_ms, bool auto_quality); + +/** Applies a "set" message; returns true if anything changed. */ +bool camera_controls_apply(camera_controls_t *controls, const char *message, size_t length); + +/** Feeds automatic quality with the outcome of each streamed frame. */ +void camera_controls_on_frame_result(camera_controls_t *controls, bool completed, uint32_t duration_ms); + +/** Restores full quality and turns the flash off when the viewer disconnects. */ +void camera_controls_viewer_left(camera_controls_t *controls); + +uint32_t camera_controls_frame_interval_ms(const camera_controls_t *controls); + +/** Heap-allocated JSON, or NULL when out of memory. The caller frees it. */ +char *camera_controls_capabilities_json(const camera_controls_t *controls); +char *camera_controls_state_json(const camera_controls_t *controls); + +/** True once after any state change the viewer has not been told about. */ +bool camera_controls_take_state_changed(camera_controls_t *controls); + +#endif /* CAMERA_CONTROLS_H */ diff --git a/examples/camera/components/webrtc_camera/idf_component.yml b/examples/camera/components/webrtc_camera/idf_component.yml new file mode 100644 index 0000000..3b1c7d7 --- /dev/null +++ b/examples/camera/components/webrtc_camera/idf_component.yml @@ -0,0 +1,10 @@ +description: "SinricPro WebRTC live view for ESP32 cameras: JPEG frames over a DataChannel" +dependencies: + idf: + version: ">=5.1" + espressif/esp_peer: + version: "^1.5.5" + espressif/esp32-camera: + version: "^2.1.7" + espressif/cjson: + version: "*" diff --git a/examples/camera/components/webrtc_camera/include/webrtc_camera.h b/examples/camera/components/webrtc_camera/include/webrtc_camera.h new file mode 100644 index 0000000..b596b4a --- /dev/null +++ b/examples/camera/components/webrtc_camera/include/webrtc_camera.h @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#ifndef WEBRTC_CAMERA_H +#define WEBRTC_CAMERA_H + +#include +#include +#include +#include "esp_camera.h" +#include "esp_err.h" +#include "freertos/FreeRTOS.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief One STUN/TURN server URL with its optional credentials + */ +typedef struct { + const char *url; /**< "stun:host:port", "turn:host:port?transport=udp", "turns:host:443?transport=tcp" */ + const char *username; /**< NULL or empty when the server needs none */ + const char *credential; /**< NULL or empty when the server needs none */ +} webrtc_camera_ice_server_t; + +/** + * @brief Microphone source for the optional PCMU audio track + * + * Fills @p size bytes of 8 kHz PCMU (160 = 20 ms) and returns true when a frame + * is ready. Called on the session task every loop, even without a viewer, so + * the source can drain its input. Must not block. + */ +typedef bool (*webrtc_camera_audio_source_t)(uint8_t *pcmu, size_t size, void *ctx); + +typedef struct { + uint32_t task_stack_size; + UBaseType_t task_priority; + uint32_t answer_timeout_ms; /**< Leaves headroom inside Alexa's 6 s answer budget */ + uint32_t channel_open_timeout_ms; /**< Drops a viewer whose DataChannel never opens */ + uint32_t frame_interval_ms; /**< Initial frame rate; viewers can change it */ + size_t max_frame_bytes; + uint32_t data_channel_send_cache; + uint32_t data_channel_recv_cache; + bool auto_quality; /**< Lower frame rate, then JPEG quality, when the link backs up */ + /** Largest viewer-selectable resolution. Must not exceed the size esp_camera_init() + * used, since the camera's JPEG buffers are sized for it. FRAMESIZE_INVALID = current size. */ + framesize_t max_frame_size; + int flash_gpio; /**< Flash LED GPIO, -1 for none (AI-Thinker ESP32-CAM: 4) */ + webrtc_camera_audio_source_t audio_source; /**< NULL = no microphone track */ + void *audio_ctx; +} webrtc_camera_config_t; + +#define WEBRTC_CAMERA_CONFIG_DEFAULT() { \ + .task_stack_size = 24 * 1024, \ + .task_priority = 4, \ + .answer_timeout_ms = 4000, \ + .channel_open_timeout_ms = 15000, \ + .frame_interval_ms = 200, \ + .max_frame_bytes = 128 * 1024, \ + .data_channel_send_cache = 48 * 1024, \ + .data_channel_recv_cache = 16 * 1024, \ + .auto_quality = true, \ + .max_frame_size = FRAMESIZE_INVALID, \ + .flash_gpio = -1, \ + .audio_source = NULL, \ + .audio_ctx = NULL, \ +} + +/** + * @brief Session handle (opaque) + * + * One viewer at a time: a new offer replaces the current viewer. Every esp_peer + * call runs on the session task; other tasks reach it only through a queue. + */ +typedef struct webrtc_camera *webrtc_camera_handle_t; + +/** + * @brief Start the session task + * + * Call after esp_camera_init() and once Wi-Fi is connected. The session lives + * for the lifetime of the application. + */ +esp_err_t webrtc_camera_start(const webrtc_camera_config_t *config, webrtc_camera_handle_t *out_handle); + +/** + * @brief Answer a viewer's SDP offer + * + * Blocks until the local answer, carrying every gathered candidate, is ready or + * the answer timeout expires. Signaling is a single exchange with no trickle ICE, + * so the answer must be complete when it is returned. + * + * @param[in] handle Session + * @param[in] offer_sdp Viewer's SDP offer + * @param[in] ice_servers STUN/TURN servers for this session + * @param[in] ice_server_count Number of servers + * @param[out] answer_sdp On success, the heap-allocated answer; the caller frees it + * @param[out] error Optional; on failure, a reason to show the viewer + * @param[in] error_size Size of @p error + * + * @return ESP_OK, or an error with @p error describing it + */ +esp_err_t webrtc_camera_handle_offer(webrtc_camera_handle_t handle, const char *offer_sdp, + const webrtc_camera_ice_server_t *ice_servers, size_t ice_server_count, + char **answer_sdp, char *error, size_t error_size); + +/** + * @brief Close the current viewer, if any + */ +void webrtc_camera_stop_viewer(webrtc_camera_handle_t handle); + +/** + * @brief Whether a viewer's DataChannel is open + */ +bool webrtc_camera_is_streaming(webrtc_camera_handle_t handle); + +#ifdef __cplusplus +} +#endif + +#endif /* WEBRTC_CAMERA_H */ diff --git a/examples/camera/components/webrtc_camera/jpeg_streamer.c b/examples/camera/components/webrtc_camera/jpeg_streamer.c new file mode 100644 index 0000000..47cf736 --- /dev/null +++ b/examples/camera/components/webrtc_camera/jpeg_streamer.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#include "jpeg_streamer.h" +#include +#include "webrtc_camera_priv.h" + +static void put_le32(uint8_t *out, uint32_t value) +{ + out[0] = (uint8_t)value; + out[1] = (uint8_t)(value >> 8); + out[2] = (uint8_t)(value >> 16); + out[3] = (uint8_t)(value >> 24); +} + +void jpeg_streamer_init(jpeg_streamer_t *streamer, uint32_t frame_interval_ms, size_t max_frame_bytes) +{ + memset(streamer, 0, sizeof(*streamer)); + streamer->frame_interval_ms = frame_interval_ms; + streamer->max_frame_bytes = max_frame_bytes; +} + +void jpeg_streamer_reset(jpeg_streamer_t *streamer) +{ + if (streamer->frame != NULL) { + esp_camera_fb_return(streamer->frame); + streamer->frame = NULL; + } + streamer->offset = 0; +} + +static jpeg_streamer_result_t finish(jpeg_streamer_t *streamer, jpeg_streamer_result_t result) +{ + streamer->last_duration_ms = webrtc_camera_now_ms() - streamer->frame_started_ms; + streamer->last_blocked_sends = streamer->blocked_sends; + streamer->last_sent_bytes = streamer->offset; + streamer->last_frame_bytes = streamer->frame != NULL ? streamer->frame->len : 0; + streamer->blocked_sends = 0; + /* reset() clears the offset and returns the frame, so the outcome is captured first. */ + jpeg_streamer_reset(streamer); + return result; +} + +jpeg_streamer_result_t jpeg_streamer_loop(jpeg_streamer_t *streamer, esp_peer_handle_t peer, uint16_t stream_id) +{ + if (streamer->frame == NULL) { + uint32_t now = webrtc_camera_now_ms(); + if (now - streamer->last_frame_ms < streamer->frame_interval_ms) { + return JPEG_STREAMER_IDLE; + } + streamer->last_frame_ms = now; + + streamer->frame = esp_camera_fb_get(); + if (streamer->frame == NULL) { + return JPEG_STREAMER_IDLE; + } + streamer->frame_started_ms = now; + streamer->offset = 0; + + /* Oversized frames count as congestion, so automatic quality compresses harder. */ + if (streamer->frame->format != PIXFORMAT_JPEG || streamer->frame->len > streamer->max_frame_bytes) { + return finish(streamer, JPEG_STREAMER_ABANDONED); + } + streamer->frame_id++; + } + + size_t remaining = streamer->frame->len - streamer->offset; + size_t bytes = remaining < JPEG_STREAMER_CHUNK_SIZE ? remaining : JPEG_STREAMER_CHUNK_SIZE; + + put_le32(streamer->packet, JPEG_STREAMER_MAGIC); + put_le32(streamer->packet + 4, streamer->frame_id); + put_le32(streamer->packet + 8, (uint32_t)streamer->frame->len); + put_le32(streamer->packet + 12, (uint32_t)streamer->offset); + memcpy(streamer->packet + JPEG_STREAMER_HEADER_SIZE, streamer->frame->buf + streamer->offset, bytes); + + esp_peer_data_frame_t data = { + .type = ESP_PEER_DATA_CHANNEL_DATA, + .stream_id = stream_id, + .data = streamer->packet, + .size = (int)(bytes + JPEG_STREAMER_HEADER_SIZE), + }; + int ret = esp_peer_send_data(peer, &data); + if (ret == ESP_PEER_ERR_NONE) { + streamer->offset += bytes; + } else if (ret == ESP_PEER_ERR_WOULD_BLOCK) { + streamer->blocked_sends++; + } + + if (streamer->offset == streamer->frame->len) { + return finish(streamer, JPEG_STREAMER_COMPLETED); + } + /* A stalled frame is abandoned so the viewer gets a fresh one instead of a late one. */ + if (webrtc_camera_now_ms() - streamer->frame_started_ms > JPEG_STREAMER_STALL_MS || + (ret != ESP_PEER_ERR_NONE && ret != ESP_PEER_ERR_WOULD_BLOCK)) { + return finish(streamer, JPEG_STREAMER_ABANDONED); + } + return JPEG_STREAMER_SENDING; +} diff --git a/examples/camera/components/webrtc_camera/jpeg_streamer.h b/examples/camera/components/webrtc_camera/jpeg_streamer.h new file mode 100644 index 0000000..8db870b --- /dev/null +++ b/examples/camera/components/webrtc_camera/jpeg_streamer.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#ifndef JPEG_STREAMER_H +#define JPEG_STREAMER_H + +#include +#include +#include +#include "esp_camera.h" +#include "esp_peer.h" + +/* Wire format per DataChannel message, shared with the SinricPro portal and app viewers: + * four little-endian uint32 (magic, frame id, total length, offset) followed by up to + * JPEG_STREAMER_CHUNK_SIZE bytes. Viewers reassemble by frame id and offset. */ +#define JPEG_STREAMER_MAGIC 0x47504A53 +#define JPEG_STREAMER_HEADER_SIZE 16 +#define JPEG_STREAMER_CHUNK_SIZE 1024 +#define JPEG_STREAMER_STALL_MS 1000 + +typedef enum { + JPEG_STREAMER_IDLE, + JPEG_STREAMER_SENDING, + JPEG_STREAMER_COMPLETED, + JPEG_STREAMER_ABANDONED, +} jpeg_streamer_result_t; + +typedef struct { + camera_fb_t *frame; + size_t offset; + uint32_t frame_id; + uint32_t frame_interval_ms; + size_t max_frame_bytes; + uint32_t last_frame_ms; + uint32_t frame_started_ms; + uint32_t blocked_sends; + + /* Outcome of the most recent finished frame. Many blocked sends with little progress + * means the DataChannel is wedged rather than slow, which quality reduction cannot fix. */ + uint32_t last_duration_ms; + uint32_t last_blocked_sends; + size_t last_sent_bytes; + size_t last_frame_bytes; + + uint8_t packet[JPEG_STREAMER_HEADER_SIZE + JPEG_STREAMER_CHUNK_SIZE]; +} jpeg_streamer_t; + +void jpeg_streamer_init(jpeg_streamer_t *streamer, uint32_t frame_interval_ms, size_t max_frame_bytes); + +/** + * Sends at most one fragment per call, so ICE and SCTP keep running between fragments. + * Call from the task that owns the peer. + */ +jpeg_streamer_result_t jpeg_streamer_loop(jpeg_streamer_t *streamer, esp_peer_handle_t peer, uint16_t stream_id); + +/** Returns any frame in progress to the camera driver. */ +void jpeg_streamer_reset(jpeg_streamer_t *streamer); + +#endif /* JPEG_STREAMER_H */ diff --git a/examples/camera/components/webrtc_camera/webrtc_camera.c b/examples/camera/components/webrtc_camera/webrtc_camera.c new file mode 100644 index 0000000..e26deb2 --- /dev/null +++ b/examples/camera/components/webrtc_camera/webrtc_camera.c @@ -0,0 +1,718 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#include "webrtc_camera.h" + +#include +#include +#include +#include +#include +#include + +#include "esp_log.h" +#include "esp_peer.h" +#include "esp_peer_default.h" +#include "esp_wifi.h" +#include "freertos/queue.h" +#include "freertos/semphr.h" +#include "freertos/task.h" + +#include "camera_controls.h" +#include "jpeg_streamer.h" +#include "webrtc_camera_priv.h" + +static const char *TAG = "webrtc_camera"; + +#define MAX_SIGNAL_BYTES (16 * 1024) +#define MAX_ICE_SERVERS 8 +#define MAX_CANDIDATES 16 +/* esp_peer reports its SDP after gathering; stray CANDIDATE messages may trail it briefly. */ +#define CANDIDATE_SETTLE_MS 200 +/* Extra wait beyond answer_timeout_ms, so the session task reports its own timeout first. */ +#define ANSWER_WAIT_SLACK_MS 500 +#define MAX_CONTROL_BYTES 512 +/* Measured on an AI-Thinker ESP32-CAM: at -85 dBm the Wi-Fi TX buffers never recycle fast + * enough and the DTLS handshake cannot complete. -75 leaves margin before that cliff. */ +#define WEAK_SIGNAL_DBM (-75) +#define AUDIO_FRAME_BYTES 160 /* 20 ms of 8 kHz PCMU */ +#define AUDIO_FRAME_MS 20 + +typedef struct { + char *url; + char *username; + char *credential; +} ice_server_t; + +typedef enum { + COMMAND_START, + COMMAND_STOP, +} command_type_t; + +typedef struct { + command_type_t type; + uint32_t sequence; + char *offer; /* owned; freed by the session task */ + ice_server_t *servers; /* owned; freed by the session task */ + size_t server_count; +} command_t; + +struct webrtc_camera { + webrtc_camera_config_t config; + jpeg_streamer_t streamer; + camera_controls_t controls; + QueueHandle_t commands; + SemaphoreHandle_t answer_ready; + SemaphoreHandle_t answer_lock; + SemaphoreHandle_t offer_lock; + TaskHandle_t task; + + /* Written only by webrtc_camera_handle_offer(), under offer_lock. */ + uint32_t sequence; + + /* Owned by the session task. */ + esp_peer_handle_t peer; + esp_peer_default_cfg_t peer_defaults; + ice_server_t *servers; + size_t server_count; + esp_peer_ice_server_cfg_t server_cfg[MAX_ICE_SERVERS]; + char *local_sdp; + char *candidates[MAX_CANDIDATES]; + size_t candidate_count; + uint32_t active_sequence; + uint32_t session_started_ms; + uint32_t last_signal_ms; + uint32_t audio_pts; + bool answer_published; + bool close_requested; + bool audio_active; + bool capabilities_pending; + bool state_pending; + uint16_t channel_id; + atomic_bool channel_open; + + /* Handed from the session task to webrtc_camera_handle_offer(); guarded by answer_lock. */ + uint32_t answer_sequence; + bool answer_ok; + char *answer; + char answer_error[128]; +}; + +static void set_error(char *error, size_t size, const char *text) +{ + if (error != NULL && size > 0) { + snprintf(error, size, "%s", text); + } +} + +/* The viewer sees this text, and a weak link is the usual cause of a failed handshake: below + * about -80 dBm the Wi-Fi driver runs out of TX buffers. Naming the signal turns an opaque + * timeout into something the user can act on. */ +static void append_weak_signal(char *error, size_t size) +{ + wifi_ap_record_t ap; + if (error == NULL || size == 0 || esp_wifi_sta_get_ap_info(&ap) != ESP_OK || ap.rssi >= WEAK_SIGNAL_DBM) { + return; + } + size_t used = strlen(error); + if (used + 1 < size) { + snprintf(error + used, size - used, " (Wi-Fi signal %d dBm is too weak)", ap.rssi); + } +} + +static void free_servers(ice_server_t *servers, size_t count) +{ + for (size_t i = 0; servers != NULL && i < count; i++) { + free(servers[i].url); + free(servers[i].username); + free(servers[i].credential); + } + free(servers); +} + +static ice_server_t *copy_servers(const webrtc_camera_ice_server_t *source, size_t count) +{ + ice_server_t *servers = calloc(count, sizeof(*servers)); + if (servers == NULL) { + return NULL; + } + for (size_t i = 0; i < count; i++) { + servers[i].url = strdup(source[i].url != NULL ? source[i].url : ""); + servers[i].username = strdup(source[i].username != NULL ? source[i].username : ""); + servers[i].credential = strdup(source[i].credential != NULL ? source[i].credential : ""); + if (servers[i].url == NULL || servers[i].username == NULL || servers[i].credential == NULL) { + free_servers(servers, i + 1); + return NULL; + } + } + return servers; +} + +static void free_command(command_t *command) +{ + free(command->offer); + free_servers(command->servers, command->server_count); + command->offer = NULL; + command->servers = NULL; + command->server_count = 0; +} + +static void clear_signals(struct webrtc_camera *session) +{ + free(session->local_sdp); + session->local_sdp = NULL; + for (size_t i = 0; i < session->candidate_count; i++) { + free(session->candidates[i]); + } + session->candidate_count = 0; +} + +/* Signaling is a single exchange, so candidates reported separately from the SDP must ride inside + * it. With BUNDLE every media section shares one transport, so the first section carries them. */ +static char *build_answer(const struct webrtc_camera *session) +{ + const char *sdp = session->local_sdp; + const char *eol = strstr(sdp, "\r\n") != NULL ? "\r\n" : "\n"; + size_t eol_len = strlen(eol); + size_t sdp_len = strlen(sdp); + + size_t extra_len = 0; + for (size_t i = 0; i < session->candidate_count; i++) { + if (strstr(sdp, session->candidates[i]) == NULL) { + extra_len += 2 + strlen(session->candidates[i]) + eol_len; + } + } + if (extra_len == 0) { + return strdup(sdp); + } + + /* "\r\nm=" plus its terminator: five bytes, which CRLF SDPs from browsers need. */ + char media_line[5]; + snprintf(media_line, sizeof(media_line), "%sm=", eol); + /* The first line is always "v=0", so every media line follows an end of line. */ + const char *first = strstr(sdp, media_line); + const char *second = first != NULL ? strstr(first + eol_len, media_line) : NULL; + + size_t split = sdp_len; + bool terminate = false; + if (second != NULL) { + split = (size_t)(second - sdp) + eol_len; + } else { + terminate = sdp_len < eol_len || strcmp(sdp + sdp_len - eol_len, eol) != 0; + } + + char *out = malloc(sdp_len + extra_len + (terminate ? eol_len : 0) + 1); + if (out == NULL) { + return NULL; + } + + char *cursor = out; + memcpy(cursor, sdp, split); + cursor += split; + if (terminate) { + memcpy(cursor, eol, eol_len); + cursor += eol_len; + } + for (size_t i = 0; i < session->candidate_count; i++) { + const char *candidate = session->candidates[i]; + if (strstr(sdp, candidate) != NULL) { + continue; + } + size_t length = strlen(candidate); + memcpy(cursor, "a=", 2); + memcpy(cursor + 2, candidate, length); + memcpy(cursor + 2 + length, eol, eol_len); + cursor += 2 + length + eol_len; + } + memcpy(cursor, sdp + split, sdp_len - split); + cursor += sdp_len - split; + *cursor = '\0'; + return out; +} + +static void publish_answer(struct webrtc_camera *session, bool ok, const char *error) +{ + char *answer = ok ? build_answer(session) : NULL; + if (ok && answer == NULL) { + ok = false; + error = "Camera is out of memory"; + } + + xSemaphoreTake(session->answer_lock, portMAX_DELAY); + /* An answer nobody collected belongs to an offer whose caller already timed out. */ + free(session->answer); + session->answer = answer; + session->answer_sequence = session->active_sequence; + session->answer_ok = ok; + snprintf(session->answer_error, sizeof(session->answer_error), "%s", error != NULL ? error : ""); + xSemaphoreGive(session->answer_lock); + + session->answer_published = true; + xSemaphoreGive(session->answer_ready); +} + +static void close_peer(struct webrtc_camera *session) +{ + jpeg_streamer_reset(&session->streamer); + camera_controls_viewer_left(&session->controls); + atomic_store(&session->channel_open, false); + session->close_requested = false; + session->audio_active = false; + session->capabilities_pending = false; + session->state_pending = false; + + if (session->peer != NULL) { + esp_peer_close(session->peer); + session->peer = NULL; + } + /* Freed only after the peer is closed: it keeps pointers to these strings. */ + free_servers(session->servers, session->server_count); + session->servers = NULL; + session->server_count = 0; + + if (!session->answer_published) { + publish_answer(session, false, "Camera closed the WebRTC session before answering"); + } +} + +static int on_msg(esp_peer_msg_t *msg, void *ctx) +{ + struct webrtc_camera *session = ctx; + if (msg == NULL || msg->data == NULL || msg->size <= 0 || msg->size > MAX_SIGNAL_BYTES) { + return ESP_PEER_ERR_INVALID_ARG; + } + + if (msg->type == ESP_PEER_MSG_TYPE_SDP) { + char *sdp = strndup((const char *)msg->data, (size_t)msg->size); + if (sdp != NULL) { + free(session->local_sdp); + session->local_sdp = sdp; + } + } else if (msg->type == ESP_PEER_MSG_TYPE_CANDIDATE && session->candidate_count < MAX_CANDIDATES) { + const char *text = (const char *)msg->data; + size_t length = (size_t)msg->size; + while (length > 0 && isspace((unsigned char)text[0])) { + text++; + length--; + } + while (length > 0 && isspace((unsigned char)text[length - 1])) { + length--; + } + if (length >= 2 && strncmp(text, "a=", 2) == 0) { + text += 2; + length -= 2; + } + if (length >= 10 && strncmp(text, "candidate:", 10) == 0) { + char *candidate = strndup(text, length); + if (candidate != NULL) { + session->candidates[session->candidate_count++] = candidate; + } + } + } + + session->last_signal_ms = webrtc_camera_now_ms(); + return ESP_PEER_ERR_NONE; +} + +static int on_state(esp_peer_state_t state, void *ctx) +{ + struct webrtc_camera *session = ctx; + ESP_LOGI(TAG, "Peer state: %d", (int)state); + if (state == ESP_PEER_STATE_DISCONNECTED || state == ESP_PEER_STATE_CONNECT_FAILED) { + session->close_requested = true; + } + return ESP_PEER_ERR_NONE; +} + +static int on_channel_open(esp_peer_data_channel_info_t *channel, void *ctx) +{ + struct webrtc_camera *session = ctx; + session->channel_id = channel->stream_id; + session->capabilities_pending = true; + session->state_pending = true; + atomic_store(&session->channel_open, true); + return ESP_PEER_ERR_NONE; +} + +static int on_channel_close(esp_peer_data_channel_info_t *channel, void *ctx) +{ + (void)channel; + struct webrtc_camera *session = ctx; + atomic_store(&session->channel_open, false); + session->close_requested = true; + return ESP_PEER_ERR_NONE; +} + +static int on_data(esp_peer_data_frame_t *frame, void *ctx) +{ + struct webrtc_camera *session = ctx; + if (frame == NULL || frame->type != ESP_PEER_DATA_CHANNEL_STRING || frame->size <= 0 || + frame->size > MAX_CONTROL_BYTES) { + return ESP_PEER_ERR_NONE; + } + camera_controls_apply(&session->controls, (const char *)frame->data, (size_t)frame->size); + session->state_pending = true; + return ESP_PEER_ERR_NONE; +} + +static void start_peer(struct webrtc_camera *session, command_t *command) +{ + close_peer(session); + + session->active_sequence = command->sequence; + session->answer_published = false; + session->session_started_ms = webrtc_camera_now_ms(); + session->last_signal_ms = session->session_started_ms; + clear_signals(session); + + /* esp_peer keeps pointers to these strings until it is closed, so the session takes them over. */ + session->servers = command->servers; + session->server_count = command->server_count; + command->servers = NULL; + command->server_count = 0; + for (size_t i = 0; i < session->server_count; i++) { + session->server_cfg[i] = (esp_peer_ice_server_cfg_t){ + .stun_url = session->servers[i].url, + .user = session->servers[i].username[0] != '\0' ? session->servers[i].username : NULL, + .psw = session->servers[i].credential[0] != '\0' ? session->servers[i].credential : NULL, + }; + } + + esp_peer_cfg_t cfg = { + .server_lists = session->server_count > 0 ? session->server_cfg : NULL, + .server_num = (uint8_t)session->server_count, + .role = ESP_PEER_ROLE_CONTROLLED, + .enable_data_channel = true, + .manual_ch_create = true, /* the viewer creates the channel */ + .no_auto_reconnect = true, + .ctx = session, + .on_msg = on_msg, + .on_state = on_state, + .on_channel_open = on_channel_open, + .on_channel_close = on_channel_close, + .on_data = on_data, + }; + + /* Viewers offer audio only when getCameraCapabilities reported it; older viewers never do. */ + session->audio_active = session->config.audio_source != NULL && strstr(command->offer, "m=audio") != NULL; + if (session->audio_active) { + cfg.audio_info = (esp_peer_audio_stream_info_t){ESP_PEER_AUDIO_CODEC_G711U, 8000, 1}; + cfg.audio_dir = ESP_PEER_MEDIA_DIR_SEND_ONLY; + } + + memset(&session->peer_defaults, 0, sizeof(session->peer_defaults)); + session->peer_defaults.agent_recv_timeout = 10; + session->peer_defaults.data_ch_cfg.send_cache_size = session->config.data_channel_send_cache; + session->peer_defaults.data_ch_cfg.recv_cache_size = session->config.data_channel_recv_cache; + /* RTP carries only the optional PCMU track, so a DataChannel-only session would otherwise + * strand memory the Wi-Fi driver needs. Zero is not an option: esp_peer reads it as its + * 400 kB default. */ + session->peer_defaults.rtp_cfg.send_pool_size = session->audio_active ? 48 * 1024 : 4 * 1024; + session->peer_defaults.rtp_cfg.send_queue_num = session->audio_active ? 64 : 8; + cfg.extra_cfg = &session->peer_defaults; + cfg.extra_size = sizeof(session->peer_defaults); + + /* Callbacks fire only inside esp_peer_main_loop(), so the first SDP reported after the offer + * is this session's answer. */ + int ret = esp_peer_open(&cfg, esp_peer_get_default_impl(), &session->peer); + if (ret == ESP_PEER_ERR_NONE) { + ret = esp_peer_new_connection(session->peer); + } + if (ret == ESP_PEER_ERR_NONE) { + esp_peer_msg_t offer = { + .type = ESP_PEER_MSG_TYPE_SDP, + .data = (uint8_t *)command->offer, + .size = (int)strlen(command->offer), + }; + ret = esp_peer_send_msg(session->peer, &offer); + } + + if (ret != ESP_PEER_ERR_NONE) { + ESP_LOGE(TAG, "Peer start failed: %d", ret); + char reason[64]; + snprintf(reason, sizeof(reason), "Camera could not start WebRTC (error %d)", ret); + publish_answer(session, false, reason); + close_peer(session); + } +} + +/* Takes ownership of json. */ +static bool send_text(struct webrtc_camera *session, char *json) +{ + if (json == NULL) { + return false; + } + esp_peer_data_frame_t frame = { + .type = ESP_PEER_DATA_CHANNEL_STRING, + .stream_id = session->channel_id, + .data = (uint8_t *)json, + .size = (int)strlen(json), + }; + bool sent = esp_peer_send_data(session->peer, &frame) == ESP_PEER_ERR_NONE; + free(json); + return sent; +} + +static void stream_to_viewer(struct webrtc_camera *session) +{ + /* Control messages go first: they are tiny, and the viewer needs them to render its controls. */ + if (session->capabilities_pending && + send_text(session, camera_controls_capabilities_json(&session->controls))) { + session->capabilities_pending = false; + } + if (!session->capabilities_pending && session->state_pending && + send_text(session, camera_controls_state_json(&session->controls))) { + session->state_pending = false; + } + + session->streamer.frame_interval_ms = camera_controls_frame_interval_ms(&session->controls); + jpeg_streamer_result_t result = jpeg_streamer_loop(&session->streamer, session->peer, session->channel_id); + if (result == JPEG_STREAMER_COMPLETED || result == JPEG_STREAMER_ABANDONED) { + if (result == JPEG_STREAMER_ABANDONED) { + ESP_LOGW(TAG, "Frame dropped: %u/%u bytes in %" PRIu32 " ms, %" PRIu32 " blocked sends", + (unsigned)session->streamer.last_sent_bytes, (unsigned)session->streamer.last_frame_bytes, + session->streamer.last_duration_ms, session->streamer.last_blocked_sends); + } + camera_controls_on_frame_result(&session->controls, result == JPEG_STREAMER_COMPLETED, + session->streamer.last_duration_ms); + } + + if (camera_controls_take_state_changed(&session->controls)) { + session->state_pending = true; + } +} + +static void poll_audio(struct webrtc_camera *session) +{ + if (session->config.audio_source == NULL) { + return; + } + + uint8_t pcmu[AUDIO_FRAME_BYTES]; + if (!session->config.audio_source(pcmu, sizeof(pcmu), session->config.audio_ctx)) { + return; + } + if (session->audio_active && atomic_load(&session->channel_open)) { + esp_peer_audio_frame_t frame = {.pts = session->audio_pts, .data = pcmu, .size = sizeof(pcmu)}; + esp_peer_send_audio(session->peer, &frame); + } + session->audio_pts += AUDIO_FRAME_MS; +} + +static void session_task(void *arg) +{ + struct webrtc_camera *session = arg; + + for (;;) { + /* With no viewer and no microphone there is nothing to poll, so wait for a command. */ + bool busy = session->peer != NULL || session->config.audio_source != NULL; + command_t command; + TickType_t wait = busy ? 0 : portMAX_DELAY; + while (xQueueReceive(session->commands, &command, wait) == pdTRUE) { + if (command.type == COMMAND_START) { + start_peer(session, &command); + } else { + close_peer(session); + } + free_command(&command); + wait = 0; + } + + poll_audio(session); + + if (session->peer != NULL) { + esp_peer_main_loop(session->peer); + + uint32_t now = webrtc_camera_now_ms(); + if (!session->answer_published) { + if (session->local_sdp != NULL && now - session->last_signal_ms >= CANDIDATE_SETTLE_MS) { + publish_answer(session, true, NULL); + } else if (now - session->session_started_ms >= session->config.answer_timeout_ms) { + ESP_LOGW(TAG, "No local SDP within %" PRIu32 " ms", session->config.answer_timeout_ms); + publish_answer(session, false, "Camera timed out gathering network candidates"); + session->close_requested = true; + } + } + + if (atomic_load(&session->channel_open)) { + stream_to_viewer(session); + } else if (session->answer_published && + now - session->session_started_ms > session->config.channel_open_timeout_ms) { + session->close_requested = true; + } + + if (session->close_requested) { + close_peer(session); + } + } + + if (session->peer != NULL || session->config.audio_source != NULL) { + /* One fragment per loop, so the tick length caps throughput; see webrtc_camera_start(). */ + vTaskDelay(1); + } + } +} + +esp_err_t webrtc_camera_start(const webrtc_camera_config_t *config, webrtc_camera_handle_t *out_handle) +{ + if (config == NULL || out_handle == NULL) { + return ESP_ERR_INVALID_ARG; + } + *out_handle = NULL; + + if (configTICK_RATE_HZ < 1000) { + ESP_LOGW(TAG, "CONFIG_FREERTOS_HZ is %d: the streamer sends one fragment per tick and needs 1000", + (int)configTICK_RATE_HZ); + } + + struct webrtc_camera *session = calloc(1, sizeof(*session)); + if (session == NULL) { + return ESP_ERR_NO_MEM; + } + + session->config = *config; + session->answer_published = true; + atomic_init(&session->channel_open, false); + jpeg_streamer_init(&session->streamer, config->frame_interval_ms, config->max_frame_bytes); + camera_controls_init(&session->controls, config->max_frame_size, config->flash_gpio, + config->frame_interval_ms, config->auto_quality); + + session->commands = xQueueCreate(4, sizeof(command_t)); + session->answer_ready = xSemaphoreCreateBinary(); + session->answer_lock = xSemaphoreCreateMutex(); + session->offer_lock = xSemaphoreCreateMutex(); + + if (session->commands != NULL && session->answer_ready != NULL && session->answer_lock != NULL && + session->offer_lock != NULL && + xTaskCreate(session_task, "webrtc_camera", config->task_stack_size, session, config->task_priority, + &session->task) == pdPASS) { + *out_handle = session; + return ESP_OK; + } + + if (session->commands != NULL) { + vQueueDelete(session->commands); + } + if (session->answer_ready != NULL) { + vSemaphoreDelete(session->answer_ready); + } + if (session->answer_lock != NULL) { + vSemaphoreDelete(session->answer_lock); + } + if (session->offer_lock != NULL) { + vSemaphoreDelete(session->offer_lock); + } + free(session); + return ESP_ERR_NO_MEM; +} + +esp_err_t webrtc_camera_handle_offer(webrtc_camera_handle_t handle, const char *offer_sdp, + const webrtc_camera_ice_server_t *ice_servers, size_t ice_server_count, + char **answer_sdp, char *error, size_t error_size) +{ + if (handle == NULL || offer_sdp == NULL || answer_sdp == NULL || + (ice_server_count > 0 && ice_servers == NULL)) { + set_error(error, error_size, "Invalid WebRTC offer"); + return ESP_ERR_INVALID_ARG; + } + *answer_sdp = NULL; + set_error(error, error_size, ""); + + if (strncmp(offer_sdp, "v=0", 3) != 0 || strlen(offer_sdp) > MAX_SIGNAL_BYTES) { + set_error(error, error_size, "Invalid WebRTC offer"); + return ESP_ERR_INVALID_ARG; + } + if (ice_server_count > MAX_ICE_SERVERS) { + ESP_LOGW(TAG, "Using the first %d of %u ICE servers", MAX_ICE_SERVERS, (unsigned)ice_server_count); + ice_server_count = MAX_ICE_SERVERS; + } + + xSemaphoreTake(handle->offer_lock, portMAX_DELAY); + + command_t command = { + .type = COMMAND_START, + .sequence = ++handle->sequence, + .offer = strdup(offer_sdp), + .servers = ice_server_count > 0 ? copy_servers(ice_servers, ice_server_count) : NULL, + .server_count = ice_server_count, + }; + /* Drop a completion left over from an earlier offer that timed out on this side. */ + xSemaphoreTake(handle->answer_ready, 0); + + if (command.offer == NULL || (ice_server_count > 0 && command.servers == NULL)) { + free_command(&command); + set_error(error, error_size, "Camera is out of memory"); + xSemaphoreGive(handle->offer_lock); + return ESP_ERR_NO_MEM; + } + if (xQueueSend(handle->commands, &command, 0) != pdTRUE) { + free_command(&command); + set_error(error, error_size, "Camera WebRTC session is busy"); + xSemaphoreGive(handle->offer_lock); + return ESP_ERR_INVALID_STATE; + } + + bool answered = false; + bool ok = false; + char *answer = NULL; + const TickType_t budget = pdMS_TO_TICKS(handle->config.answer_timeout_ms + ANSWER_WAIT_SLACK_MS); + const TickType_t started = xTaskGetTickCount(); + + while (!answered) { + TickType_t elapsed = xTaskGetTickCount() - started; + if (elapsed >= budget || xSemaphoreTake(handle->answer_ready, budget - elapsed) != pdTRUE) { + break; + } + + xSemaphoreTake(handle->answer_lock, portMAX_DELAY); + if (handle->answer_sequence == command.sequence) { + answered = true; + ok = handle->answer_ok; + answer = handle->answer; + handle->answer = NULL; + if (!ok) { + set_error(error, error_size, handle->answer_error); + } + } + xSemaphoreGive(handle->answer_lock); + } + + if (!answered) { + set_error(error, error_size, "Camera timed out creating the WebRTC answer"); + } else if (ok && (answer == NULL || answer[0] == '\0')) { + set_error(error, error_size, "Camera produced an empty WebRTC answer"); + ok = false; + } + if (!ok) { + free(answer); + answer = NULL; + append_weak_signal(error, error_size); + } + + xSemaphoreGive(handle->offer_lock); + + if (!ok) { + return answered ? ESP_FAIL : ESP_ERR_TIMEOUT; + } + *answer_sdp = answer; + return ESP_OK; +} + +void webrtc_camera_stop_viewer(webrtc_camera_handle_t handle) +{ + if (handle == NULL) { + return; + } + command_t command = {.type = COMMAND_STOP}; + xQueueSend(handle->commands, &command, 0); +} + +bool webrtc_camera_is_streaming(webrtc_camera_handle_t handle) +{ + return handle != NULL && atomic_load(&handle->channel_open); +} diff --git a/examples/camera/components/webrtc_camera/webrtc_camera_priv.h b/examples/camera/components/webrtc_camera/webrtc_camera_priv.h new file mode 100644 index 0000000..ef735fe --- /dev/null +++ b/examples/camera/components/webrtc_camera/webrtc_camera_priv.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#ifndef WEBRTC_CAMERA_PRIV_H +#define WEBRTC_CAMERA_PRIV_H + +#include +#include "esp_timer.h" + +/* Wraps after ~49 days; every use is a difference of two readings, which survives the wrap. */ +static inline uint32_t webrtc_camera_now_ms(void) +{ + return (uint32_t)(esp_timer_get_time() / 1000); +} + +#endif /* WEBRTC_CAMERA_PRIV_H */ diff --git a/examples/camera/main/CMakeLists.txt b/examples/camera/main/CMakeLists.txt new file mode 100644 index 0000000..ceb98e9 --- /dev/null +++ b/examples/camera/main/CMakeLists.txt @@ -0,0 +1,2 @@ +idf_component_register(SRCS "camera_example.c" "camera_boards.c" + INCLUDE_DIRS ".") diff --git a/examples/camera/main/Kconfig.projbuild b/examples/camera/main/Kconfig.projbuild new file mode 100644 index 0000000..37581d3 --- /dev/null +++ b/examples/camera/main/Kconfig.projbuild @@ -0,0 +1,58 @@ +menu "SinricPro Camera Example" + + choice CAMERA_BOARD + prompt "Camera board" + default CAMERA_BOARD_AI_THINKER if IDF_TARGET_ESP32 + default CAMERA_BOARD_XIAO_S3_SENSE if IDF_TARGET_ESP32S3 + help + Selects the camera pin mapping. The processor alone does not + identify the wiring, so choose the actual board. + + config CAMERA_BOARD_AI_THINKER + bool "AI-Thinker ESP32-CAM" + depends on IDF_TARGET_ESP32 + config CAMERA_BOARD_ESP_EYE + bool "ESP-EYE" + depends on IDF_TARGET_ESP32 + config CAMERA_BOARD_M5CAMERA + bool "M5Camera model A" + depends on IDF_TARGET_ESP32 + config CAMERA_BOARD_M5CAMERA_B + bool "M5Camera model B" + depends on IDF_TARGET_ESP32 + config CAMERA_BOARD_WROVER_KIT + bool "ESP-WROVER-KIT" + depends on IDF_TARGET_ESP32 + config CAMERA_BOARD_LILYGO_CAMERA + bool "LILYGO TTGO T-Camera (camera only)" + depends on IDF_TARGET_ESP32 + config CAMERA_BOARD_XIAO_S3_SENSE + bool "XIAO ESP32S3 Sense" + depends on IDF_TARGET_ESP32S3 + config CAMERA_BOARD_FREENOVE_S3 + bool "Freenove ESP32-S3 camera board" + depends on IDF_TARGET_ESP32S3 + config CAMERA_BOARD_ESP32S3_WROOM + bool "ESP32-S3 WROOM camera wiring (PWDN GPIO38)" + depends on IDF_TARGET_ESP32S3 + config CAMERA_BOARD_ESP32S3_GOOUUU + bool "GOOUUU ESP32-S3 camera wiring" + depends on IDF_TARGET_ESP32S3 + endchoice + + config CAMERA_FLASH_GPIO + int "Flash LED GPIO (-1 for none)" + range -1 48 + default 4 if CAMERA_BOARD_AI_THINKER + default -1 + help + Exposes a flash toggle to viewers when set. + + config CAMERA_MICROPHONE + bool "Stream the onboard PDM microphone" + depends on CAMERA_BOARD_XIAO_S3_SENSE + default y + help + Sends an 8 kHz PCMU audio track alongside the video. + +endmenu diff --git a/examples/camera/main/camera_boards.c b/examples/camera/main/camera_boards.c new file mode 100644 index 0000000..0ad41ff --- /dev/null +++ b/examples/camera/main/camera_boards.c @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + * + * Pin mappings follow Espressif's camera_pinout.h and LilyGO/esp32-camera-bme280. + * ESP32-S3 WROOM camera boards do not all share one layout: check PWDN too. + */ + +#include "camera_boards.h" +#include "sdkconfig.h" +#include "driver/gpio.h" + +camera_config_t camera_board_config(void) +{ + camera_config_t camera = { + .pin_pwdn = -1, + .pin_reset = -1, + .xclk_freq_hz = 20000000, + .ledc_timer = LEDC_TIMER_0, + .ledc_channel = LEDC_CHANNEL_0, + .pixel_format = PIXFORMAT_JPEG, + .frame_size = FRAMESIZE_VGA, + .jpeg_quality = 16, + .fb_count = 2, + .fb_location = CAMERA_FB_IN_PSRAM, + .grab_mode = CAMERA_GRAB_LATEST, + }; + +#if CONFIG_CAMERA_BOARD_AI_THINKER + camera.pin_d0 = 5; + camera.pin_d1 = 18; + camera.pin_d2 = 19; + camera.pin_d3 = 21; + camera.pin_d4 = 36; + camera.pin_d5 = 39; + camera.pin_d6 = 34; + camera.pin_d7 = 35; + camera.pin_xclk = 0; + camera.pin_sccb_sda = 26; + camera.pin_sccb_scl = 27; + camera.pin_vsync = 25; + camera.pin_href = 23; + camera.pin_pclk = 22; + camera.pin_pwdn = 32; +#elif CONFIG_CAMERA_BOARD_ESP_EYE + camera.pin_d0 = 34; + camera.pin_d1 = 13; + camera.pin_d2 = 14; + camera.pin_d3 = 35; + camera.pin_d4 = 39; + camera.pin_d5 = 38; + camera.pin_d6 = 37; + camera.pin_d7 = 36; + camera.pin_xclk = 4; + camera.pin_sccb_sda = 18; + camera.pin_sccb_scl = 23; + camera.pin_vsync = 5; + camera.pin_href = 27; + camera.pin_pclk = 25; +#elif CONFIG_CAMERA_BOARD_M5CAMERA || CONFIG_CAMERA_BOARD_M5CAMERA_B + camera.pin_d0 = 32; + camera.pin_d1 = 35; + camera.pin_d2 = 34; + camera.pin_d3 = 5; + camera.pin_d4 = 39; + camera.pin_d5 = 18; + camera.pin_d6 = 36; + camera.pin_d7 = 19; + camera.pin_xclk = 27; + camera.pin_sccb_scl = 23; + camera.pin_href = 26; + camera.pin_pclk = 21; + camera.pin_reset = 15; +#if CONFIG_CAMERA_BOARD_M5CAMERA + camera.pin_sccb_sda = 25; + camera.pin_vsync = 22; +#else + camera.pin_sccb_sda = 22; + camera.pin_vsync = 25; +#endif +#elif CONFIG_CAMERA_BOARD_WROVER_KIT + camera.pin_d0 = 4; + camera.pin_d1 = 5; + camera.pin_d2 = 18; + camera.pin_d3 = 19; + camera.pin_d4 = 36; + camera.pin_d5 = 39; + camera.pin_d6 = 34; + camera.pin_d7 = 35; + camera.pin_xclk = 21; + camera.pin_sccb_sda = 26; + camera.pin_sccb_scl = 27; + camera.pin_vsync = 25; + camera.pin_href = 23; + camera.pin_pclk = 22; +#elif CONFIG_CAMERA_BOARD_LILYGO_CAMERA + camera.pin_d0 = 5; + camera.pin_d1 = 14; + camera.pin_d2 = 4; + camera.pin_d3 = 15; + camera.pin_d4 = 18; + camera.pin_d5 = 23; + camera.pin_d6 = 36; + camera.pin_d7 = 39; + camera.pin_xclk = 32; + camera.pin_sccb_sda = 13; + camera.pin_sccb_scl = 12; + camera.pin_vsync = 27; + camera.pin_href = 25; + camera.pin_pclk = 19; + camera.pin_pwdn = 26; +#elif CONFIG_CAMERA_BOARD_XIAO_S3_SENSE + camera.pin_d0 = 15; + camera.pin_d1 = 17; + camera.pin_d2 = 18; + camera.pin_d3 = 16; + camera.pin_d4 = 14; + camera.pin_d5 = 12; + camera.pin_d6 = 11; + camera.pin_d7 = 48; + camera.pin_xclk = 10; + camera.pin_sccb_sda = 40; + camera.pin_sccb_scl = 39; + camera.pin_vsync = 38; + camera.pin_href = 47; + camera.pin_pclk = 13; +#elif CONFIG_CAMERA_BOARD_FREENOVE_S3 || CONFIG_CAMERA_BOARD_ESP32S3_WROOM || CONFIG_CAMERA_BOARD_ESP32S3_GOOUUU + camera.pin_d0 = 11; + camera.pin_d1 = 9; + camera.pin_d2 = 8; + camera.pin_d3 = 10; + camera.pin_d4 = 12; + camera.pin_d5 = 18; + camera.pin_d6 = 17; + camera.pin_d7 = 16; + camera.pin_xclk = 15; + camera.pin_sccb_sda = 4; + camera.pin_sccb_scl = 5; + camera.pin_vsync = 6; + camera.pin_href = 7; + camera.pin_pclk = 13; +#if CONFIG_CAMERA_BOARD_ESP32S3_WROOM + camera.pin_pwdn = 38; +#endif +#endif + + return camera; +} + +const char *camera_board_name(void) +{ +#if CONFIG_CAMERA_BOARD_AI_THINKER + return "AI-Thinker ESP32-CAM"; +#elif CONFIG_CAMERA_BOARD_ESP_EYE + return "ESP-EYE"; +#elif CONFIG_CAMERA_BOARD_M5CAMERA + return "M5Camera A"; +#elif CONFIG_CAMERA_BOARD_M5CAMERA_B + return "M5Camera B"; +#elif CONFIG_CAMERA_BOARD_WROVER_KIT + return "ESP-WROVER-KIT"; +#elif CONFIG_CAMERA_BOARD_LILYGO_CAMERA + return "LILYGO T-Camera (camera only)"; +#elif CONFIG_CAMERA_BOARD_XIAO_S3_SENSE + return "XIAO ESP32S3 Sense"; +#elif CONFIG_CAMERA_BOARD_FREENOVE_S3 + return "Freenove ESP32-S3"; +#elif CONFIG_CAMERA_BOARD_ESP32S3_WROOM + return "ESP32-S3 WROOM (PWDN 38)"; +#elif CONFIG_CAMERA_BOARD_ESP32S3_GOOUUU + return "GOOUUU ESP32-S3"; +#else + return "Unknown camera board"; +#endif +} + +void camera_board_prepare(void) +{ +#if CONFIG_CAMERA_BOARD_ESP_EYE + /* ESP-EYE's camera does not probe without pull-ups on these pins. */ + gpio_config_t pull_ups = { + .pin_bit_mask = BIT64(GPIO_NUM_13) | BIT64(GPIO_NUM_14), + .mode = GPIO_MODE_INPUT, + .pull_up_en = GPIO_PULLUP_ENABLE, + }; + gpio_config(&pull_ups); +#endif +} diff --git a/examples/camera/main/camera_boards.h b/examples/camera/main/camera_boards.h new file mode 100644 index 0000000..f19ae5c --- /dev/null +++ b/examples/camera/main/camera_boards.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#ifndef CAMERA_BOARDS_H +#define CAMERA_BOARDS_H + +#include "esp_camera.h" + +/** Pin mapping and capture defaults for the board selected in menuconfig. */ +camera_config_t camera_board_config(void); + +const char *camera_board_name(void); + +/** Board-specific GPIO setup that must happen before esp_camera_init(). */ +void camera_board_prepare(void); + +#endif /* CAMERA_BOARDS_H */ diff --git a/examples/camera/main/camera_example.c b/examples/camera/main/camera_example.c new file mode 100644 index 0000000..39baf30 --- /dev/null +++ b/examples/camera/main/camera_example.c @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + * + * SinricPro camera with live view in the SinricPro portal and app. Video is JPEG + * over a WebRTC DataChannel; signaling runs through SinricPro (getWebRTCAnswer), + * and STUN/TURN servers arrive with each offer, so viewing works outside the LAN. + * Viewers can change resolution and frame rate and toggle flash, flip and mirror. + * + * Portal setup: device type Camera -> Camera Stream Configuration: Board "ESP32", + * Streaming Protocol "WebRTC". Alexa and Google Home streaming is not supported. + */ + +#include +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/event_groups.h" +#include "esp_camera.h" +#include "esp_event.h" +#include "esp_heap_caps.h" +#include "esp_log.h" +#include "esp_psram.h" +#include "esp_system.h" +#include "esp_wifi.h" +#include "nvs_flash.h" + +#include "sinricpro.h" +#include "sinricpro_camera.h" +#include "webrtc_camera.h" +#include "camera_boards.h" + +#ifdef CONFIG_CAMERA_MICROPHONE +#include "driver/i2s_pdm.h" +#endif + +/* WiFi Configuration - MODIFY THESE */ +#define WIFI_SSID "WIFI_SSID" +#define WIFI_PASS "WIFI_PASS" + +/* SinricPro Configuration - MODIFY THESE */ +#define DEVICE_ID "DEVICE_ID" /* 24-character hex string */ +#define APP_KEY "APP_KEY" /* From SinricPro portal */ +#define APP_SECRET "APP_SECRET" /* From SinricPro portal */ + +/* The server sends one STUN URL and a few TURN URLs; the session uses at most 8. */ +#define MAX_ICE_SERVERS 8 + +static const char *TAG = "camera_example"; + +static EventGroupHandle_t s_wifi_event_group; +#define WIFI_CONNECTED_BIT BIT0 + +static webrtc_camera_handle_t s_session = NULL; +static sinricpro_device_handle_t s_camera = NULL; + +/* =========================================================================== + * WiFi + * =========================================================================== */ + +static void wifi_event_handler(void *arg, esp_event_base_t event_base, + int32_t event_id, void *event_data) +{ + if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_STA_START) { + esp_wifi_connect(); + } else if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_STA_DISCONNECTED) { + ESP_LOGI(TAG, "WiFi disconnected, retrying..."); + xEventGroupClearBits(s_wifi_event_group, WIFI_CONNECTED_BIT); + esp_wifi_connect(); + } else if (event_base == IP_EVENT && event_id == IP_EVENT_STA_GOT_IP) { + ip_event_got_ip_t *event = (ip_event_got_ip_t *)event_data; + ESP_LOGI(TAG, "Got IP:" IPSTR, IP2STR(&event->ip_info.ip)); + xEventGroupSetBits(s_wifi_event_group, WIFI_CONNECTED_BIT); + } +} + +static void wifi_init_sta(void) +{ + s_wifi_event_group = xEventGroupCreate(); + + ESP_ERROR_CHECK(esp_netif_init()); + ESP_ERROR_CHECK(esp_event_loop_create_default()); + esp_netif_create_default_wifi_sta(); + + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + ESP_ERROR_CHECK(esp_wifi_init(&cfg)); + + ESP_ERROR_CHECK(esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID, + &wifi_event_handler, NULL, NULL)); + ESP_ERROR_CHECK(esp_event_handler_instance_register(IP_EVENT, IP_EVENT_STA_GOT_IP, + &wifi_event_handler, NULL, NULL)); + + wifi_config_t wifi_config = { + .sta = { + .ssid = WIFI_SSID, + .password = WIFI_PASS, + .threshold.authmode = WIFI_AUTH_WPA2_PSK, + }, + }; + ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA)); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_STA, &wifi_config)); + ESP_ERROR_CHECK(esp_wifi_start()); + /* Modem sleep delays every outgoing packet, which a video stream cannot absorb. */ + esp_wifi_set_ps(WIFI_PS_NONE); + + xEventGroupWaitBits(s_wifi_event_group, WIFI_CONNECTED_BIT, pdFALSE, pdFALSE, portMAX_DELAY); + + wifi_ap_record_t ap; + if (esp_wifi_sta_get_ap_info(&ap) == ESP_OK) { + ESP_LOGI(TAG, "Connected to %s, RSSI %d dBm", WIFI_SSID, ap.rssi); + if (ap.rssi < -75) { + ESP_LOGW(TAG, "Live view needs about -75 dBm or better; move the camera closer to the access point"); + } + } +} + +/* =========================================================================== + * Camera and microphone + * =========================================================================== */ + +static esp_err_t camera_init(void) +{ + if (!esp_psram_is_initialized()) { + ESP_LOGE(TAG, "PSRAM is not available; camera frame buffers and WebRTC need it"); + return ESP_ERR_NO_MEM; + } + + camera_board_prepare(); + camera_config_t config = camera_board_config(); + /* Frame buffers are sized for the init resolution, so initialise at the largest + * size viewers may pick and start streaming smaller. */ + config.frame_size = FRAMESIZE_SVGA; + + esp_err_t err = esp_camera_init(&config); + if (err != ESP_OK) { + ESP_LOGE(TAG, "Camera init failed: 0x%x (%s). Check the board selected in menuconfig.", + err, esp_err_to_name(err)); + return err; + } + + sensor_t *sensor = esp_camera_sensor_get(); + sensor->set_framesize(sensor, FRAMESIZE_QVGA); + ESP_LOGI(TAG, "Camera: %s, PSRAM: %u bytes", camera_board_name(), (unsigned)esp_psram_get_size()); + return ESP_OK; +} + +#ifdef CONFIG_CAMERA_MICROPHONE +static i2s_chan_handle_t s_microphone = NULL; + +static esp_err_t microphone_init(void) +{ + i2s_chan_config_t channel_config = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_0, I2S_ROLE_MASTER); + esp_err_t err = i2s_new_channel(&channel_config, NULL, &s_microphone); + if (err != ESP_OK) { + return err; + } + + /* XIAO ESP32S3 Sense: PDM clock on GPIO42, data on GPIO41. */ + i2s_pdm_rx_config_t pdm_config = { + .clk_cfg = I2S_PDM_RX_CLK_DEFAULT_CONFIG(16000), + .slot_cfg = I2S_PDM_RX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO), + .gpio_cfg = { + .clk = GPIO_NUM_42, + .din = GPIO_NUM_41, + }, + }; + err = i2s_channel_init_pdm_rx_mode(s_microphone, &pdm_config); + return err == ESP_OK ? i2s_channel_enable(s_microphone) : err; +} + +static uint8_t encode_mulaw(int16_t pcm) +{ + int value = pcm; + int sign = value < 0 ? 0x80 : 0; + if (value < 0) { + value = -value; + } + if (value > 32635) { + value = 32635; + } + value += 0x84; + int exponent = 7; + for (int mask = 0x4000; exponent > 0 && !(value & mask); mask >>= 1) { + exponent--; + } + return (uint8_t)~(sign | (exponent << 4) | ((value >> (exponent + 3)) & 0x0F)); +} + +/* 20 ms of 16 kHz PDM audio, averaged down to the 8 kHz PCMU the WebRTC audio track carries. */ +static bool read_microphone(uint8_t *pcmu, size_t size, void *ctx) +{ + static int16_t pcm[320]; + static size_t used = 0; + + size_t bytes = 0; + i2s_channel_read(s_microphone, (uint8_t *)pcm + used, sizeof(pcm) - used, &bytes, 0); + used += bytes; + if (used < sizeof(pcm)) { + return false; + } + used = 0; + + for (size_t i = 0; i < size && 2 * i + 1 < 320; i++) { + pcmu[i] = encode_mulaw((int16_t)(((int32_t)pcm[2 * i] + pcm[2 * i + 1]) / 2)); + } + return true; +} +#endif + +/* =========================================================================== + * SinricPro callbacks + * =========================================================================== */ + +static bool on_webrtc_offer(const char *device_id, const char *offer_sdp, + const sinricpro_ice_server_t *ice_servers, size_t ice_server_count, + char **answer_sdp, void *user_data) +{ + /* Field for field the same; the SDK and the session component stay independent of each other. */ + webrtc_camera_ice_server_t servers[MAX_ICE_SERVERS]; + size_t count = ice_server_count < MAX_ICE_SERVERS ? ice_server_count : MAX_ICE_SERVERS; + for (size_t i = 0; i < count; i++) { + servers[i] = (webrtc_camera_ice_server_t){ + .url = ice_servers[i].url, + .username = ice_servers[i].username, + .credential = ice_servers[i].credential, + }; + } + + ESP_LOGI(TAG, "WebRTC offer for %s with %u ICE server URLs", device_id, (unsigned)count); + + char error[160]; + esp_err_t err = webrtc_camera_handle_offer(s_session, offer_sdp, servers, count, + answer_sdp, error, sizeof(error)); + if (err != ESP_OK) { + ESP_LOGW(TAG, "WebRTC answer failed: %s", error); + /* Shown to the viewer in the SinricPro app and portal. */ + sinricpro_set_response_message(error); + return false; + } + + ESP_LOGI(TAG, "WebRTC answer sent"); + return true; +} + +static bool on_power_state(const char *device_id, bool *state, void *user_data) +{ + ESP_LOGI(TAG, "PowerState: %s", *state ? "ON" : "OFF"); + if (!*state) { + webrtc_camera_stop_viewer(s_session); + } + return true; +} + +static void sinricpro_event_handler(void *arg, esp_event_base_t event_base, + int32_t event_id, void *event_data) +{ + if (event_id == SINRICPRO_EVENT_CONNECTED) { + ESP_LOGI(TAG, "Connected to SinricPro server"); + } else if (event_id == SINRICPRO_EVENT_DISCONNECTED) { + ESP_LOGW(TAG, "Disconnected from SinricPro server"); + } +} + +/* Internal RAM is the pool that runs out first: Wi-Fi, TLS and DTLS draw on it where PSRAM + * cannot substitute. A healthy total beside a small largest block is fragmentation rather + * than exhaustion, and the two need different fixes. */ +static void log_stats(void) +{ + wifi_ap_record_t ap; + int rssi = esp_wifi_sta_get_ap_info(&ap) == ESP_OK ? ap.rssi : 0; + + ESP_LOGI(TAG, "Heap free %u (min %u), internal %u (largest %u), PSRAM free %u, RSSI %d, streaming: %s", + (unsigned)esp_get_free_heap_size(), (unsigned)esp_get_minimum_free_heap_size(), + (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL), + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL), + (unsigned)heap_caps_get_free_size(MALLOC_CAP_SPIRAM), rssi, + webrtc_camera_is_streaming(s_session) ? "yes" : "no"); +} + +/* =========================================================================== + * Main Application + * =========================================================================== */ + +void app_main(void) +{ + ESP_LOGI(TAG, "SinricPro ESP-IDF Camera Example, SDK %s", sinricpro_get_version()); + + esp_err_t ret = nvs_flash_init(); + if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_ERROR_CHECK(nvs_flash_erase()); + ret = nvs_flash_init(); + } + ESP_ERROR_CHECK(ret); + + if (camera_init() != ESP_OK) { + return; + } + +#ifdef CONFIG_CAMERA_MICROPHONE + if (microphone_init() != ESP_OK) { + ESP_LOGE(TAG, "Microphone initialization failed"); + return; + } +#endif + + wifi_init_sta(); + + webrtc_camera_config_t session_config = WEBRTC_CAMERA_CONFIG_DEFAULT(); + session_config.max_frame_size = FRAMESIZE_SVGA; /* the size camera_init() allocated for */ + session_config.flash_gpio = CONFIG_CAMERA_FLASH_GPIO; +#if CONFIG_IDF_TARGET_ESP32 + /* Once Wi-Fi and the SinricPro TLS socket are up, classic ESP32 has little contiguous + * internal RAM left. Caches large enough to consume it leave the Wi-Fi driver unable to + * allocate TX buffers, and the DTLS handshake never completes. */ + session_config.data_channel_send_cache = 6 * 1024; + session_config.data_channel_recv_cache = 3 * 1024; +#endif +#ifdef CONFIG_CAMERA_MICROPHONE + session_config.audio_source = read_microphone; +#endif + + ret = webrtc_camera_start(&session_config, &s_session); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to start the WebRTC session: %s", esp_err_to_name(ret)); + return; + } + + ESP_ERROR_CHECK(esp_event_handler_register(SINRICPRO_EVENT, ESP_EVENT_ANY_ID, + &sinricpro_event_handler, NULL)); + + sinricpro_config_t sinric_config = { + .app_key = APP_KEY, + .app_secret = APP_SECRET, + .auto_reconnect = true, + .reconnect_interval_ms = 5000, + .heartbeat_interval_ms = 0, + }; + ret = sinricpro_init(&sinric_config); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to initialize SinricPro: %s", esp_err_to_name(ret)); + return; + } + + s_camera = sinricpro_camera_create(DEVICE_ID); + if (s_camera == NULL) { + ESP_LOGE(TAG, "Failed to create camera device"); + return; + } + sinricpro_camera_on_power_state(s_camera, on_power_state, NULL); + sinricpro_camera_on_webrtc_offer(s_camera, on_webrtc_offer, NULL); +#ifdef CONFIG_CAMERA_MICROPHONE + /* Viewers request an audio track only when this is set. */ + sinricpro_camera_enable_webrtc_audio(s_camera, true); +#endif + + ret = sinricpro_start(); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to start SinricPro: %s", esp_err_to_name(ret)); + return; + } + + while (1) { + vTaskDelay(pdMS_TO_TICKS(30000)); + log_stats(); + } +} diff --git a/examples/camera/main/idf_component.yml b/examples/camera/main/idf_component.yml new file mode 100644 index 0000000..be8aaca --- /dev/null +++ b/examples/camera/main/idf_component.yml @@ -0,0 +1,8 @@ +dependencies: + idf: + version: ">=5.1" + espressif/esp_websocket_client: + version: "^1.2.0" + espressif/cjson: + version: "*" +# sinricpro/esp-idf: ^1.3.0 diff --git a/examples/camera/partitions.csv b/examples/camera/partitions.csv new file mode 100644 index 0000000..e897076 --- /dev/null +++ b/examples/camera/partitions.csv @@ -0,0 +1,4 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 0x3F0000, diff --git a/examples/camera/sdkconfig.defaults b/examples/camera/sdkconfig.defaults new file mode 100644 index 0000000..e6064ad --- /dev/null +++ b/examples/camera/sdkconfig.defaults @@ -0,0 +1,20 @@ +# Camera frame buffers, and Wi-Fi, lwIP and mbedTLS allocations, go to PSRAM. +# Moving the Wi-Fi and lwIP buffers out of internal RAM is what keeps classic +# ESP32's scarce contiguous internal memory free for the driver's TX path. +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=256 +CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y +CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y + +# The streamer sends one DataChannel fragment per tick. +CONFIG_FREERTOS_HZ=1000 + +# WebRTC transport: DTLS-SRTP, and a self-signed certificate for the DTLS fingerprint. +CONFIG_MBEDTLS_SSL_PROTO_DTLS=y +CONFIG_MBEDTLS_SSL_DTLS_SRTP=y +CONFIG_MBEDTLS_X509_CREATE_C=y + +# esp_peer and the camera driver do not fit the default 1 MB app partition. +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" diff --git a/examples/camera/sdkconfig.defaults.esp32s3 b/examples/camera/sdkconfig.defaults.esp32s3 new file mode 100644 index 0000000..ca42088 --- /dev/null +++ b/examples/camera/sdkconfig.defaults.esp32s3 @@ -0,0 +1,4 @@ +# XIAO ESP32S3 Sense and the Freenove N8R8 carry octal PSRAM. A board with quad +# PSRAM needs CONFIG_SPIRAM_MODE_QUAD instead, or PSRAM fails to start at boot. +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y diff --git a/include/sinricpro.h b/include/sinricpro.h index bf5a2fb..daf5d7f 100644 --- a/include/sinricpro.h +++ b/include/sinricpro.h @@ -145,6 +145,23 @@ bool sinricpro_local_control_is_running(void); */ uint32_t sinricpro_get_timestamp(void); +/** + * @brief Set the message returned with the response to the request being handled + * + * Replaces the default "OK" / "Device did not handle request", so a client can + * show why a request failed, e.g. "Camera timed out creating the WebRTC answer". + * + * @param[in] message Message text; truncated to 191 characters + * + * @return + * - ESP_OK: Success + * - SINRICPRO_ERR_INVALID_ARG: NULL message + * + * @note Call only from inside a device callback. It applies to the response + * being built for that request and is cleared before the next one. + */ +esp_err_t sinricpro_set_response_message(const char *message); + /** * @brief Get version string * diff --git a/include/sinricpro_camera.h b/include/sinricpro_camera.h new file mode 100644 index 0000000..837e844 --- /dev/null +++ b/include/sinricpro_camera.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#ifndef SINRICPRO_CAMERA_H +#define SINRICPRO_CAMERA_H + +#include +#include +#include "sinricpro.h" +#include "sinricpro_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief PowerState callback function signature + * + * @param[in] device_id Device ID string + * @param[in,out] state Requested power state in; actual state out + * @param[in] user_data User data pointer passed during registration + * + * @return true if request handled successfully, false otherwise + */ +typedef bool (*sinricpro_camera_power_state_callback_t)( + const char *device_id, + bool *state, + void *user_data +); + +/** + * @brief One STUN/TURN server URL with its credentials, as sent by the SinricPro server + */ +typedef struct { + const char *url; /**< "stun:…", "turn:…?transport=udp" or "turns:…:443?transport=tcp" */ + const char *username; /**< Empty string when the server sent none */ + const char *credential; /**< Empty string when the server sent none */ +} sinricpro_ice_server_t; + +/** + * @brief WebRTC offer callback function signature + * + * Invoked when a viewer in the SinricPro portal or app requests live view. + * Signaling is one offer/answer exchange with no trickle ICE, so the answer + * must already contain every local candidate. + * + * @param[in] device_id Device ID string + * @param[in] offer_sdp Viewer's SDP offer (plain text, candidates included) + * @param[in] ice_servers STUN/TURN servers for this session, one entry per URL + * @param[in] ice_server_count Number of entries in @p ice_servers + * @param[out] answer_sdp Set to a malloc()ed SDP answer; the SDK frees it + * @param[in] user_data User data pointer passed during registration + * + * @return true if an answer was produced + * + * @note Unlike other callbacks this one may block while ICE gathers, for up to + * about 5 s. The strings in @p ice_servers are valid only during the call. + * @note Call sinricpro_set_response_message() before returning false to show + * the viewer why live view failed. + */ +typedef bool (*sinricpro_camera_webrtc_offer_callback_t)( + const char *device_id, + const char *offer_sdp, + const sinricpro_ice_server_t *ice_servers, + size_t ice_server_count, + char **answer_sdp, + void *user_data +); + +/** + * @brief Create a camera device + * + * @param[in] device_id Device ID string (exactly 24 hexadecimal characters) + * + * @return Device handle, or NULL on failure + * + * @note Call after sinricpro_init() and before sinricpro_start() + */ +sinricpro_device_handle_t sinricpro_camera_create(const char *device_id); + +/** + * @brief Register PowerState callback + * + * @return ESP_OK, or ESP_ERR_INVALID_ARG for a NULL device or callback + */ +esp_err_t sinricpro_camera_on_power_state( + sinricpro_device_handle_t device, + sinricpro_camera_power_state_callback_t callback, + void *user_data +); + +/** + * @brief Register the WebRTC offer callback + * + * Registering it is what makes the camera report WebRTC support through + * getCameraCapabilities, so viewers only attempt live view once it is set. + * + * @return ESP_OK, or ESP_ERR_INVALID_ARG for a NULL device or callback + */ +esp_err_t sinricpro_camera_on_webrtc_offer( + sinricpro_device_handle_t device, + sinricpro_camera_webrtc_offer_callback_t callback, + void *user_data +); + +/** + * @brief Declare that WebRTC sessions carry a microphone audio track + * + * Reported through getCameraCapabilities; viewers request an audio track in + * their offer only when it is set. + * + * @return ESP_OK, or ESP_ERR_INVALID_ARG for a NULL device + */ +esp_err_t sinricpro_camera_enable_webrtc_audio(sinricpro_device_handle_t device, bool enabled); + +/** + * @brief Send PowerState event to server + * + * @return ESP_OK when queued, or an error (see sinricpro_switch_send_power_state_event()) + */ +esp_err_t sinricpro_camera_send_power_state_event( + sinricpro_device_handle_t device, + bool state, + const char *cause +); + +/** + * @brief Delete camera device + * + * @return ESP_OK, or ESP_ERR_INVALID_ARG for a NULL device + */ +esp_err_t sinricpro_camera_delete(sinricpro_device_handle_t device); + +#ifdef __cplusplus +} +#endif + +#endif /* SINRICPRO_CAMERA_H */ diff --git a/src/capabilities/camera_controller.c b/src/capabilities/camera_controller.c new file mode 100644 index 0000000..625703f --- /dev/null +++ b/src/capabilities/camera_controller.c @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#include "camera_controller.h" +#include +#include +#include "esp_log.h" +#include "mbedtls/base64.h" + +static const char *TAG = "camera_ctrl"; + +#define ACTION_GET_CAMERA_CAPABILITIES "getCameraCapabilities" +#define ACTION_GET_WEBRTC_ANSWER "getWebRTCAnswer" + +/* The server sends one STUN URL and a few TURN URLs; this only bounds a malformed list. */ +#define MAX_ICE_SERVER_URLS 16 + +/** + * @brief CameraController context + */ +struct sinricpro_camera_controller { + sinricpro_camera_webrtc_offer_callback_t offer_callback; + void *offer_user_data; + bool webrtc_audio; +}; + +sinricpro_camera_controller_handle_t sinricpro_camera_controller_create(void) +{ + sinricpro_camera_controller_handle_t handle = calloc(1, sizeof(struct sinricpro_camera_controller)); + if (handle == NULL) { + ESP_LOGE(TAG, "Failed to allocate CameraController"); + return NULL; + } + + ESP_LOGD(TAG, "CameraController created"); + return handle; +} + +esp_err_t sinricpro_camera_controller_set_webrtc_offer_callback( + sinricpro_camera_controller_handle_t handle, + sinricpro_camera_webrtc_offer_callback_t callback, + void *user_data) +{ + if (handle == NULL || callback == NULL) { + return ESP_ERR_INVALID_ARG; + } + + handle->offer_callback = callback; + handle->offer_user_data = user_data; + return ESP_OK; +} + +void sinricpro_camera_controller_set_webrtc_audio(sinricpro_camera_controller_handle_t handle, + bool enabled) +{ + if (handle != NULL) { + handle->webrtc_audio = enabled; + } +} + +bool sinricpro_camera_controller_owns_action(const char *action) +{ + return action != NULL && + (strcmp(action, ACTION_GET_CAMERA_CAPABILITIES) == 0 || + strcmp(action, ACTION_GET_WEBRTC_ANSWER) == 0); +} + +static char *base64_decode_string(const char *input) +{ + size_t input_len = strlen(input); + size_t output_len = 0; + + /* Given no buffer, mbedTLS reports the size it needs through output_len. */ + int ret = mbedtls_base64_decode(NULL, 0, &output_len, (const unsigned char *)input, input_len); + if (ret != 0 && ret != MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL) { + return NULL; + } + + char *output = malloc(output_len + 1); + if (output == NULL) { + return NULL; + } + if (mbedtls_base64_decode((unsigned char *)output, output_len, &output_len, + (const unsigned char *)input, input_len) != 0) { + free(output); + return NULL; + } + output[output_len] = '\0'; + return output; +} + +static char *base64_encode_string(const char *input) +{ + size_t input_len = strlen(input); + size_t output_len = 0; + + /* The size reported includes the terminating NUL that mbedTLS writes. */ + mbedtls_base64_encode(NULL, 0, &output_len, (const unsigned char *)input, input_len); + if (output_len == 0) { + return NULL; + } + + char *output = malloc(output_len); + if (output == NULL) { + return NULL; + } + size_t written = 0; + if (mbedtls_base64_encode((unsigned char *)output, output_len, &written, + (const unsigned char *)input, input_len) != 0) { + free(output); + return NULL; + } + return output; +} + +/* The server sends RTCIceServer entries whose "urls" is either a string or an + * array. Each URL becomes one flat entry, so callbacks handle a single shape. */ +static size_t collect_ice_servers(cJSON *list, sinricpro_ice_server_t *out, size_t max) +{ + size_t count = 0; + cJSON *server = NULL; + + cJSON_ArrayForEach(server, list) { + if (!cJSON_IsObject(server)) { + continue; + } + + cJSON *username = cJSON_GetObjectItem(server, "username"); + cJSON *credential = cJSON_GetObjectItem(server, "credential"); + const char *user = cJSON_IsString(username) ? username->valuestring : ""; + const char *secret = cJSON_IsString(credential) ? credential->valuestring : ""; + + cJSON *urls = cJSON_GetObjectItem(server, "urls"); + if (cJSON_IsString(urls)) { + if (count < max) { + out[count++] = (sinricpro_ice_server_t){urls->valuestring, user, secret}; + } + continue; + } + + cJSON *url = NULL; + cJSON_ArrayForEach(url, urls) { + if (cJSON_IsString(url) && count < max) { + out[count++] = (sinricpro_ice_server_t){url->valuestring, user, secret}; + } + } + } + + return count; +} + +static bool handle_webrtc_offer(sinricpro_camera_controller_handle_t handle, + const char *device_id, + cJSON *request_value, + cJSON *response_value) +{ + if (handle->offer_callback == NULL) { + ESP_LOGW(TAG, "No WebRTC offer callback registered"); + return false; + } + + cJSON *offer_item = cJSON_GetObjectItem(request_value, "offer"); + if (!cJSON_IsString(offer_item)) { + ESP_LOGE(TAG, "getWebRTCAnswer request has no offer"); + return false; + } + + char *offer = base64_decode_string(offer_item->valuestring); + if (offer == NULL) { + ESP_LOGE(TAG, "WebRTC offer is not valid base64"); + return false; + } + + sinricpro_ice_server_t servers[MAX_ICE_SERVER_URLS]; + size_t server_count = collect_ice_servers(cJSON_GetObjectItem(request_value, "iceServers"), + servers, MAX_ICE_SERVER_URLS); + + char *answer_sdp = NULL; + bool success = handle->offer_callback(device_id, offer, servers, server_count, + &answer_sdp, handle->offer_user_data); + free(offer); + + if (success) { + char *answer = (answer_sdp != NULL && answer_sdp[0] != '\0') ? base64_encode_string(answer_sdp) : NULL; + if (answer != NULL) { + cJSON_AddStringToObject(response_value, "answer", answer); + free(answer); + } else { + ESP_LOGE(TAG, "WebRTC offer callback reported success without an answer"); + success = false; + } + } + + free(answer_sdp); + return success; +} + +bool sinricpro_camera_controller_handle_request( + sinricpro_camera_controller_handle_t handle, + const char *device_id, + const char *action, + cJSON *request_value, + cJSON *response_value) +{ + if (handle == NULL || action == NULL) { + return false; + } + + if (strcmp(action, ACTION_GET_CAMERA_CAPABILITIES) == 0) { + /* Viewers ask before connecting, so firmware without a WebRTC callback is + * told apart from one that is merely unreachable. */ + bool webrtc = handle->offer_callback != NULL; + cJSON_AddBoolToObject(response_value, "webrtc", webrtc); + cJSON_AddBoolToObject(response_value, "webrtcAudio", webrtc && handle->webrtc_audio); + return true; + } + + if (strcmp(action, ACTION_GET_WEBRTC_ANSWER) == 0) { + return handle_webrtc_offer(handle, device_id, request_value, response_value); + } + + return false; +} + +void sinricpro_camera_controller_destroy(sinricpro_camera_controller_handle_t handle) +{ + free(handle); +} diff --git a/src/capabilities/camera_controller.h b/src/capabilities/camera_controller.h new file mode 100644 index 0000000..fe84c45 --- /dev/null +++ b/src/capabilities/camera_controller.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#ifndef CAMERA_CONTROLLER_H +#define CAMERA_CONTROLLER_H + +#include "sinricpro_types.h" +#include "sinricpro_camera.h" +#include "cJSON.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief CameraController handle (opaque) + */ +typedef struct sinricpro_camera_controller* sinricpro_camera_controller_handle_t; + +/** + * @brief Create CameraController + * + * @return Controller handle, or NULL on failure + */ +sinricpro_camera_controller_handle_t sinricpro_camera_controller_create(void); + +/** + * @brief Set WebRTC offer callback + * + * @return ESP_OK on success + */ +esp_err_t sinricpro_camera_controller_set_webrtc_offer_callback( + sinricpro_camera_controller_handle_t handle, + sinricpro_camera_webrtc_offer_callback_t callback, + void *user_data); + +/** + * @brief Set whether WebRTC sessions carry a microphone audio track + */ +void sinricpro_camera_controller_set_webrtc_audio(sinricpro_camera_controller_handle_t handle, + bool enabled); + +/** + * @brief Whether @p action belongs to this controller + * + * Lets a device report a failed camera request as a failure rather than as an + * action nobody handled. + */ +bool sinricpro_camera_controller_owns_action(const char *action); + +/** + * @brief Handle a camera request + * + * @param[in] handle Controller handle + * @param[in] device_id Device ID + * @param[in] action Action name + * @param[in] request_value Request value JSON + * @param[in,out] response_value Response value JSON + * + * @return true if handled successfully, false otherwise + */ +bool sinricpro_camera_controller_handle_request( + sinricpro_camera_controller_handle_t handle, + const char *device_id, + const char *action, + cJSON *request_value, + cJSON *response_value); + +/** + * @brief Destroy CameraController + */ +void sinricpro_camera_controller_destroy(sinricpro_camera_controller_handle_t handle); + +#ifdef __cplusplus +} +#endif + +#endif /* CAMERA_CONTROLLER_H */ diff --git a/src/core/sinricpro_core.c b/src/core/sinricpro_core.c index f5280c2..73d57d2 100644 --- a/src/core/sinricpro_core.c +++ b/src/core/sinricpro_core.c @@ -44,6 +44,10 @@ static struct { * single caller. */ SemaphoreHandle_t dispatch_mutex; TaskHandle_t send_task; + /* Overrides the message of the response being built. Cleared before each + * handler runs and written only from inside one, which dispatch_mutex + * serialises. */ + char response_message[192]; } core_state = {0}; /* Forward declarations */ @@ -273,6 +277,7 @@ static void handle_request(cJSON *json_message, const sinricpro_msg_origin_t *or cJSON_AddItemToObject(response_payload, "value", response_value); bool success = false; + core_state.response_message[0] = '\0'; if (device != NULL && device->request_handler != NULL) { /* Call device request handler */ @@ -284,7 +289,10 @@ static void handle_request(cJSON *json_message, const sinricpro_msg_origin_t *or } cJSON_AddBoolToObject(response_payload, "success", success); - cJSON_AddStringToObject(response_payload, "message", success ? "OK" : "Device did not handle request"); + const char *message = core_state.response_message[0] != '\0' + ? core_state.response_message + : (success ? "OK" : "Device did not handle request"); + cJSON_AddStringToObject(response_payload, "message", message); /* Queued with the origin it must go back on, so the send path can route it * without knowing which peer happens to be talking to us now. */ @@ -849,6 +857,16 @@ uint32_t sinricpro_get_timestamp(void) return core_state.timestamp; } +esp_err_t sinricpro_set_response_message(const char *message) +{ + if (message == NULL) { + return SINRICPRO_ERR_INVALID_ARG; + } + + strlcpy(core_state.response_message, message, sizeof(core_state.response_message)); + return ESP_OK; +} + const char* sinricpro_get_version(void) { return SINRICPRO_VERSION; diff --git a/src/core/sinricpro_frame_assembler.c b/src/core/sinricpro_frame_assembler.c new file mode 100644 index 0000000..7f86504 --- /dev/null +++ b/src/core/sinricpro_frame_assembler.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#include "sinricpro_frame_assembler.h" +#include +#include + +void sinricpro_frame_assembler_init(sinricpro_frame_assembler_t *assembler, size_t max_size) +{ + memset(assembler, 0, sizeof(*assembler)); + assembler->max_size = max_size; +} + +void sinricpro_frame_assembler_reset(sinricpro_frame_assembler_t *assembler) +{ + assembler->length = 0; + assembler->expected = 0; + assembler->discarding = false; +} + +void sinricpro_frame_assembler_free(sinricpro_frame_assembler_t *assembler) +{ + free(assembler->buffer); + assembler->buffer = NULL; + assembler->capacity = 0; + sinricpro_frame_assembler_reset(assembler); +} + +sinricpro_frame_result_t sinricpro_frame_assembler_feed(sinricpro_frame_assembler_t *assembler, + const char *data, size_t data_len, + size_t payload_offset, size_t payload_len, + const char **message, size_t *message_len) +{ + if (payload_offset == 0) { + /* A new message: whatever was in progress can no longer be completed. */ + sinricpro_frame_assembler_reset(assembler); + if (payload_len == 0) { + payload_len = data_len; + } + if (payload_len == 0) { + return SINRICPRO_FRAME_PENDING; + } + + assembler->expected = payload_len; + if (payload_len > assembler->max_size) { + assembler->discarding = true; + } else if (payload_len + 1 > assembler->capacity) { + char *grown = realloc(assembler->buffer, payload_len + 1); + if (grown == NULL) { + assembler->discarding = true; + } else { + assembler->buffer = grown; + assembler->capacity = payload_len + 1; + } + } + } else if (assembler->expected == 0 || payload_len != assembler->expected || + payload_offset != assembler->length) { + sinricpro_frame_assembler_reset(assembler); + return SINRICPRO_FRAME_DROPPED; + } + + if (data_len > assembler->expected - assembler->length) { + sinricpro_frame_assembler_reset(assembler); + return SINRICPRO_FRAME_DROPPED; + } + + if (!assembler->discarding) { + memcpy(assembler->buffer + assembler->length, data, data_len); + } + assembler->length += data_len; + + if (assembler->length < assembler->expected) { + return SINRICPRO_FRAME_PENDING; + } + + /* A dropped message is reported once, on its last chunk, rather than per chunk. */ + bool discarded = assembler->discarding; + size_t length = assembler->length; + sinricpro_frame_assembler_reset(assembler); + if (discarded) { + return SINRICPRO_FRAME_DROPPED; + } + + assembler->buffer[length] = '\0'; + *message = assembler->buffer; + *message_len = length; + return SINRICPRO_FRAME_COMPLETE; +} diff --git a/src/core/sinricpro_frame_assembler.h b/src/core/sinricpro_frame_assembler.h new file mode 100644 index 0000000..2bce545 --- /dev/null +++ b/src/core/sinricpro_frame_assembler.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#ifndef SINRICPRO_FRAME_ASSEMBLER_H +#define SINRICPRO_FRAME_ASSEMBLER_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Result of adding one received chunk + */ +typedef enum { + SINRICPRO_FRAME_PENDING, /**< More chunks of this message are expected */ + SINRICPRO_FRAME_COMPLETE, /**< A whole message is available */ + SINRICPRO_FRAME_DROPPED, /**< Too large, out of memory, or chunks out of sequence */ +} sinricpro_frame_result_t; + +/** + * @brief Reassembles websocket messages delivered in pieces + * + * esp_websocket_client posts a frame larger than its buffer as several data + * events, each carrying the frame's total length and its own offset. Parsing + * each piece on its own hands the core truncated JSON. + */ +typedef struct { + char *buffer; + size_t capacity; + size_t length; /**< Bytes received for the message in progress */ + size_t expected; /**< Total length of the message in progress; 0 when idle */ + size_t max_size; + bool discarding; /**< The message in progress is being dropped */ +} sinricpro_frame_assembler_t; + +void sinricpro_frame_assembler_init(sinricpro_frame_assembler_t *assembler, size_t max_size); + +/** + * @brief Abandon any partially received message, keeping the buffer + */ +void sinricpro_frame_assembler_reset(sinricpro_frame_assembler_t *assembler); + +void sinricpro_frame_assembler_free(sinricpro_frame_assembler_t *assembler); + +/** + * @brief Add one received chunk + * + * @param[in] assembler Assembler + * @param[in] data Chunk bytes + * @param[in] data_len Chunk length + * @param[in] payload_offset Offset of this chunk within the message + * @param[in] payload_len Total message length; 0 means the chunk is the whole message + * @param[out] message On SINRICPRO_FRAME_COMPLETE, the NUL-terminated message, + * owned by the assembler and valid until the next call + * @param[out] message_len On SINRICPRO_FRAME_COMPLETE, its length + */ +sinricpro_frame_result_t sinricpro_frame_assembler_feed(sinricpro_frame_assembler_t *assembler, + const char *data, size_t data_len, + size_t payload_offset, size_t payload_len, + const char **message, size_t *message_len); + +#ifdef __cplusplus +} +#endif + +#endif /* SINRICPRO_FRAME_ASSEMBLER_H */ diff --git a/src/core/sinricpro_websocket.c b/src/core/sinricpro_websocket.c index 787c013..1033126 100644 --- a/src/core/sinricpro_websocket.c +++ b/src/core/sinricpro_websocket.c @@ -8,10 +8,12 @@ #include "sinricpro_websocket.h" #include "sinricpro.h" +#include "sinricpro_frame_assembler.h" #include #include #include "esp_log.h" #include "esp_websocket_client.h" +#include "esp_transport_ws.h" #include "esp_netif.h" #include "esp_wifi.h" #include "esp_crt_bundle.h" @@ -30,6 +32,8 @@ static struct { bool initialized; bool connected; SemaphoreHandle_t mutex; + /* Only touched from the websocket task, which delivers events in order. */ + sinricpro_frame_assembler_t assembler; } ws_state = {0}; /** @@ -43,6 +47,7 @@ static void websocket_event_handler(void *arg, esp_event_base_t event_base, switch (event_id) { case WEBSOCKET_EVENT_CONNECTED: ESP_LOGI(TAG, "WebSocket connected"); + sinricpro_frame_assembler_reset(&ws_state.assembler); xSemaphoreTake(ws_state.mutex, portMAX_DELAY); ws_state.connected = true; xSemaphoreGive(ws_state.mutex); @@ -54,6 +59,8 @@ static void websocket_event_handler(void *arg, esp_event_base_t event_base, case WEBSOCKET_EVENT_DISCONNECTED: ESP_LOGI(TAG, "WebSocket disconnected"); + /* A message cut off by the disconnect can never be completed. */ + sinricpro_frame_assembler_reset(&ws_state.assembler); xSemaphoreTake(ws_state.mutex, portMAX_DELAY); ws_state.connected = false; xSemaphoreGive(ws_state.mutex); @@ -63,27 +70,33 @@ static void websocket_event_handler(void *arg, esp_event_base_t event_base, } break; - case WEBSOCKET_EVENT_DATA: - ESP_LOGD(TAG, "WebSocket data received (len=%d)", data->data_len); - - if (data->data_len > 0 && data->data_ptr != NULL) { - /* Null-terminate the data */ - char *message = malloc(data->data_len + 1); - if (message) { - memcpy(message, data->data_ptr, data->data_len); - message[data->data_len] = '\0'; - - if (ws_state.callbacks.on_receive) { - ws_state.callbacks.on_receive(message, data->data_len, - ws_state.callbacks.context); - } - - free(message); - } else { - ESP_LOGE(TAG, "Failed to allocate memory for received message"); - } + case WEBSOCKET_EVENT_DATA: { + ESP_LOGD(TAG, "WebSocket data received (len=%d, offset=%d, total=%d)", + data->data_len, data->payload_offset, data->payload_len); + + /* Ping, pong and close frames carry no SinricPro message. Every piece of + * a text frame is posted with the frame's own opcode. */ + if (data->op_code != WS_TRANSPORT_OPCODES_TEXT || data->data_len < 0 || + data->payload_offset < 0 || data->payload_len < 0) { + break; + } + + const char *message = NULL; + size_t message_len = 0; + sinricpro_frame_result_t result = sinricpro_frame_assembler_feed( + &ws_state.assembler, data->data_ptr, (size_t)data->data_len, + (size_t)data->payload_offset, (size_t)data->payload_len, + &message, &message_len); + + if (result == SINRICPRO_FRAME_DROPPED) { + ESP_LOGE(TAG, "Dropped a %d byte message: above CONFIG_SINRICPRO_MAX_MESSAGE_SIZE (%d), " + "out of memory, or received out of sequence", + data->payload_len, CONFIG_SINRICPRO_MAX_MESSAGE_SIZE); + } else if (result == SINRICPRO_FRAME_COMPLETE && ws_state.callbacks.on_receive) { + ws_state.callbacks.on_receive(message, message_len, ws_state.callbacks.context); } break; + } case WEBSOCKET_EVENT_ERROR: ESP_LOGE(TAG, "WebSocket error"); @@ -167,6 +180,8 @@ esp_err_t sinricpro_ws_init(const char *server_url, /* Save callbacks */ memcpy(&ws_state.callbacks, callbacks, sizeof(sinricpro_ws_callbacks_t)); + sinricpro_frame_assembler_init(&ws_state.assembler, CONFIG_SINRICPRO_MAX_MESSAGE_SIZE); + /* Configure WebSocket client */ esp_websocket_client_config_t ws_config = { .uri = ws_state.uri, @@ -267,6 +282,9 @@ esp_err_t sinricpro_ws_deinit(void) ws_state.client = NULL; } + /* The client task is gone, so nothing can feed the assembler any more. */ + sinricpro_frame_assembler_free(&ws_state.assembler); + /* Free URI */ if (ws_state.uri) { free(ws_state.uri); diff --git a/src/devices/sinricpro_camera.c b/src/devices/sinricpro_camera.c new file mode 100644 index 0000000..f984a96 --- /dev/null +++ b/src/devices/sinricpro_camera.c @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * This file is part of the SinricPro ESP-IDF component + * (https://github.com/sinricpro/esp-idf) + */ + +#include "sinricpro_camera.h" +#include "../core/sinricpro_device_internal.h" +#include "../capabilities/power_state_controller.h" +#include "../capabilities/camera_controller.h" +#include +#include +#include "esp_log.h" + +static const char *TAG = "sinricpro_camera"; + +/** + * @brief Camera device structure + */ +typedef struct { + sinricpro_device_t base; /* Base device structure */ + sinricpro_power_state_controller_handle_t power_state; + sinricpro_camera_controller_handle_t camera; +} sinricpro_camera_device_t; + +/** + * @brief Request handler for camera device + */ +static bool camera_request_handler(const char *device_id, + const char *action, + const char *instance_id, + cJSON *request_value, + cJSON *response_value, + void *user_data) +{ + sinricpro_camera_device_t *device = (sinricpro_camera_device_t *)user_data; + + ESP_LOGD(TAG, "Camera request: device=%s, action=%s", device_id, action); + + if (sinricpro_camera_controller_owns_action(action)) { + return sinricpro_camera_controller_handle_request(device->camera, + device_id, + action, + request_value, + response_value); + } + + if (sinricpro_power_state_controller_handle_request(device->power_state, + device_id, + action, + request_value, + response_value)) { + return true; + } + + ESP_LOGW(TAG, "Unhandled action: %s", action); + return false; +} + +sinricpro_device_handle_t sinricpro_camera_create(const char *device_id) +{ + if (device_id == NULL) { + ESP_LOGE(TAG, "Invalid device_id"); + return NULL; + } + + size_t id_len = strlen(device_id); + if (id_len == 0 || id_len >= CONFIG_SINRICPRO_MAX_DEVICE_ID_LEN) { + ESP_LOGE(TAG, "Invalid device_id length: %zu", id_len); + return NULL; + } + + sinricpro_camera_device_t *device = calloc(1, sizeof(sinricpro_camera_device_t)); + if (device == NULL) { + ESP_LOGE(TAG, "Failed to allocate camera device"); + return NULL; + } + + strncpy(device->base.device_id, device_id, sizeof(device->base.device_id) - 1); + device->base.device_type = SINRICPRO_DEVICE_TYPE_CAMERA; + device->base.request_handler = camera_request_handler; + device->base.user_data = device; + device->base.next = NULL; + + device->power_state = sinricpro_power_state_controller_create(); + device->camera = sinricpro_camera_controller_create(); + if (device->power_state == NULL || device->camera == NULL) { + ESP_LOGE(TAG, "Failed to create camera capabilities"); + sinricpro_camera_controller_destroy(device->camera); + if (device->power_state) { + sinricpro_power_state_controller_destroy(device->power_state); + } + free(device); + return NULL; + } + + esp_err_t ret = sinricpro_core_register_device(&device->base); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to register device: %s", esp_err_to_name(ret)); + sinricpro_camera_controller_destroy(device->camera); + sinricpro_power_state_controller_destroy(device->power_state); + free(device); + return NULL; + } + + ESP_LOGI(TAG, "Camera device created: %s", device_id); + + return (sinricpro_device_handle_t)device; +} + +esp_err_t sinricpro_camera_on_power_state(sinricpro_device_handle_t handle, + sinricpro_camera_power_state_callback_t callback, + void *user_data) +{ + if (handle == NULL || callback == NULL) { + return ESP_ERR_INVALID_ARG; + } + + sinricpro_camera_device_t *device = (sinricpro_camera_device_t *)handle; + + return sinricpro_power_state_controller_set_callback(device->power_state, + callback, + user_data); +} + +esp_err_t sinricpro_camera_on_webrtc_offer(sinricpro_device_handle_t handle, + sinricpro_camera_webrtc_offer_callback_t callback, + void *user_data) +{ + if (handle == NULL || callback == NULL) { + return ESP_ERR_INVALID_ARG; + } + + sinricpro_camera_device_t *device = (sinricpro_camera_device_t *)handle; + + return sinricpro_camera_controller_set_webrtc_offer_callback(device->camera, + callback, + user_data); +} + +esp_err_t sinricpro_camera_enable_webrtc_audio(sinricpro_device_handle_t handle, bool enabled) +{ + if (handle == NULL) { + return ESP_ERR_INVALID_ARG; + } + + sinricpro_camera_device_t *device = (sinricpro_camera_device_t *)handle; + sinricpro_camera_controller_set_webrtc_audio(device->camera, enabled); + return ESP_OK; +} + +esp_err_t sinricpro_camera_send_power_state_event(sinricpro_device_handle_t handle, + bool state, + const char *cause) +{ + if (handle == NULL || cause == NULL) { + return ESP_ERR_INVALID_ARG; + } + + sinricpro_camera_device_t *device = (sinricpro_camera_device_t *)handle; + + return sinricpro_power_state_controller_send_event(device->power_state, + device->base.device_id, + state, + cause); +} + +esp_err_t sinricpro_camera_delete(sinricpro_device_handle_t handle) +{ + if (handle == NULL) { + return ESP_ERR_INVALID_ARG; + } + + sinricpro_camera_device_t *device = (sinricpro_camera_device_t *)handle; + + esp_err_t ret = sinricpro_core_unregister_device(device->base.device_id); + if (ret != ESP_OK) { + ESP_LOGW(TAG, "Failed to unregister device: %s", esp_err_to_name(ret)); + } + + sinricpro_camera_controller_destroy(device->camera); + sinricpro_power_state_controller_destroy(device->power_state); + free(device); + + ESP_LOGI(TAG, "Camera device deleted"); + + return ESP_OK; +} diff --git a/test/host/run.sh b/test/host/run.sh index ad56d9b..bb45dd8 100644 --- a/test/host/run.sh +++ b/test/host/run.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# Build and run the host tests for the signing / verification wire contract. +# Build and run the host tests for the SDK's wire contracts: message signing and +# verification, websocket message reassembly, and camera WebRTC signaling. # -# Compiles the real src/core/sinricpro_signature.c off-target against shim -# headers (OpenSSL stands in for mbedTLS), so what is tested is the shipped -# source rather than a re-implementation of it. +# Compiles the real sources off-target against shim headers (OpenSSL stands in +# for mbedTLS), so what is tested is the shipped source rather than a +# re-implementation of it. # # test/host/run.sh # @@ -25,11 +26,14 @@ fi out="$(mktemp -d)" trap 'rm -rf "$out"' EXIT -gcc -std=c11 -Wall -Wextra -Wno-unused-parameter -O1 -g \ - -I "$here/shims" \ - -I "$root/src/core" \ - -I "$root/include" \ - -I "$cjson_dir" \ +cflags=(-std=c11 -Wall -Wextra -Wno-unused-parameter -O1 -g + -I "$here/shims" + -I "$root/src/core" + -I "$root/src/capabilities" + -I "$root/include" + -I "$cjson_dir") + +gcc "${cflags[@]}" \ "$here/test_signature.c" \ "$here/shims/shims.c" \ "$root/src/core/sinricpro_signature.c" \ @@ -37,4 +41,22 @@ gcc -std=c11 -Wall -Wextra -Wno-unused-parameter -O1 -g \ -lcrypto -lm \ -o "$out/test_signature" -"$out/test_signature" +gcc "${cflags[@]}" \ + "$here/test_frame_assembler.c" \ + "$root/src/core/sinricpro_frame_assembler.c" \ + -o "$out/test_frame_assembler" + +gcc "${cflags[@]}" \ + "$here/test_camera_controller.c" \ + "$here/shims/shims.c" \ + "$root/src/capabilities/camera_controller.c" \ + "$cjson_dir/cJSON.c" \ + -lcrypto -lm \ + -o "$out/test_camera_controller" + +status=0 +for test in test_signature test_frame_assembler test_camera_controller; do + echo "== $test" + "$out/$test" || status=1 +done +exit "$status" diff --git a/test/host/shims/esp_event.h b/test/host/shims/esp_event.h new file mode 100644 index 0000000..5bd8195 --- /dev/null +++ b/test/host/shims/esp_event.h @@ -0,0 +1,9 @@ +/* Host-test shim: sinricpro.h declares its event base; nothing under test posts events. */ +#ifndef SINRICPRO_HOST_SHIM_ESP_EVENT_H +#define SINRICPRO_HOST_SHIM_ESP_EVENT_H + +typedef const char *esp_event_base_t; + +#define ESP_EVENT_DECLARE_BASE(id) extern esp_event_base_t const id + +#endif diff --git a/test/host/shims/mbedtls/base64.h b/test/host/shims/mbedtls/base64.h index 1948688..99cc15e 100644 --- a/test/host/shims/mbedtls/base64.h +++ b/test/host/shims/mbedtls/base64.h @@ -1,10 +1,16 @@ -/* Host-test shim: mbedTLS base64 surface used by sinricpro_signature.c. */ +/* Host-test shim: mbedTLS base64 surface used by sinricpro_signature.c and camera_controller.c. */ #ifndef SINRICPRO_HOST_SHIM_MBEDTLS_BASE64_H #define SINRICPRO_HOST_SHIM_MBEDTLS_BASE64_H #include +#define MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL -0x002A +#define MBEDTLS_ERR_BASE64_INVALID_CHARACTER -0x002C + int mbedtls_base64_encode(unsigned char *dst, size_t dlen, size_t *olen, const unsigned char *src, size_t slen); +int mbedtls_base64_decode(unsigned char *dst, size_t dlen, size_t *olen, + const unsigned char *src, size_t slen); + #endif diff --git a/test/host/shims/shims.c b/test/host/shims/shims.c index 46e2882..64c2ebb 100644 --- a/test/host/shims/shims.c +++ b/test/host/shims/shims.c @@ -122,3 +122,71 @@ int mbedtls_base64_encode(unsigned char *dst, size_t dlen, size_t *olen, return 0; } + +static int base64_value(unsigned char c) +{ + if (c >= 'A' && c <= 'Z') { + return c - 'A'; + } + if (c >= 'a' && c <= 'z') { + return c - 'a' + 26; + } + if (c >= '0' && c <= '9') { + return c - '0' + 52; + } + if (c == '+') { + return 62; + } + return c == '/' ? 63 : -1; +} + +/* Mirrors mbedTLS: an empty input decodes to nothing, and a NULL or short + * buffer reports the size needed through olen with BUFFER_TOO_SMALL. */ +int mbedtls_base64_decode(unsigned char *dst, size_t dlen, size_t *olen, + const unsigned char *src, size_t slen) +{ + if (slen == 0) { + *olen = 0; + return 0; + } + if (slen % 4 != 0) { + return MBEDTLS_ERR_BASE64_INVALID_CHARACTER; + } + + size_t pad = 0; + while (pad < 2 && src[slen - 1 - pad] == '=') { + pad++; + } + for (size_t i = 0; i < slen - pad; i++) { + if (base64_value(src[i]) < 0) { + return MBEDTLS_ERR_BASE64_INVALID_CHARACTER; + } + } + + size_t need = slen / 4 * 3 - pad; + if (dst == NULL || dlen < need) { + *olen = need; + return MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL; + } + + size_t o = 0; + for (size_t i = 0; i < slen; i += 4) { + unsigned v = 0; + for (int k = 0; k < 4; k++) { + unsigned char c = src[i + k]; + v = (v << 6) | (c == '=' ? 0u : (unsigned)base64_value(c)); + } + if (o < need) { + dst[o++] = (unsigned char)(v >> 16); + } + if (o < need) { + dst[o++] = (unsigned char)(v >> 8); + } + if (o < need) { + dst[o++] = (unsigned char)v; + } + } + + *olen = need; + return 0; +} diff --git a/test/host/test_camera_controller.c b/test/host/test_camera_controller.c new file mode 100644 index 0000000..df6563b --- /dev/null +++ b/test/host/test_camera_controller.c @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * Host tests for the camera WebRTC signaling contract shared with the portal, + * the app and the Arduino SDK: the offer arrives base64-encoded, ICE servers + * arrive as RTCIceServer entries whose "urls" is a string or an array, and the + * answer goes back base64-encoded. + * + * Build and run: test/host/run.sh + */ + +#define _POSIX_C_SOURCE 200809L + +#include "camera_controller.h" +#include "mbedtls/base64.h" +#include "cJSON.h" + +#include +#include +#include + +static int failures; + +#define CHECK(cond, ...) \ + do { \ + if (cond) { \ + printf(" ok " __VA_ARGS__); \ + printf("\n"); \ + } else { \ + failures++; \ + printf(" FAIL " __VA_ARGS__); \ + printf(" (%s:%d)\n", __FILE__, __LINE__); \ + } \ + } while (0) + +#define MAX_SEEN 16 + +/* What the offer callback received, and what it should return. */ +static struct { + int calls; + char offer[256]; + size_t server_count; + char url[MAX_SEEN][96]; + char username[MAX_SEEN][48]; + char credential[MAX_SEEN][48]; + const char *answer; + bool result; +} seen; + +static bool offer_callback(const char *device_id, const char *offer_sdp, + const sinricpro_ice_server_t *servers, size_t count, + char **answer_sdp, void *user_data) +{ + seen.calls++; + snprintf(seen.offer, sizeof(seen.offer), "%s", offer_sdp); + seen.server_count = count; + for (size_t i = 0; i < count && i < MAX_SEEN; i++) { + snprintf(seen.url[i], sizeof(seen.url[i]), "%s", servers[i].url); + snprintf(seen.username[i], sizeof(seen.username[i]), "%s", servers[i].username); + snprintf(seen.credential[i], sizeof(seen.credential[i]), "%s", servers[i].credential); + } + *answer_sdp = seen.answer != NULL ? strdup(seen.answer) : NULL; + return seen.result; +} + +static char *encode(const char *text) +{ + size_t length = 0; + mbedtls_base64_encode(NULL, 0, &length, (const unsigned char *)text, strlen(text)); + char *out = malloc(length); + mbedtls_base64_encode((unsigned char *)out, length, &length, (const unsigned char *)text, strlen(text)); + return out; +} + +/* Runs one request through the controller; the caller deletes *response. */ +static bool run(sinricpro_camera_controller_handle_t controller, const char *action, + const char *request_json, cJSON **response) +{ + cJSON *request = request_json != NULL ? cJSON_Parse(request_json) : cJSON_CreateObject(); + *response = cJSON_CreateObject(); + bool handled = sinricpro_camera_controller_handle_request(controller, "5dc1564130xxxxxxxxxxxxxx", + action, request, *response); + cJSON_Delete(request); + return handled; +} + +static void test_capabilities(void) +{ + printf("getCameraCapabilities\n"); + + sinricpro_camera_controller_handle_t controller = sinricpro_camera_controller_create(); + cJSON *response = NULL; + + bool handled = run(controller, "getCameraCapabilities", NULL, &response); + CHECK(handled, "answered even without a WebRTC callback"); + CHECK(cJSON_IsFalse(cJSON_GetObjectItem(response, "webrtc")) && + cJSON_IsFalse(cJSON_GetObjectItem(response, "webrtcAudio")), + "firmware without a callback reports no WebRTC, so viewers ask for an update"); + cJSON_Delete(response); + + sinricpro_camera_controller_set_webrtc_audio(controller, true); + run(controller, "getCameraCapabilities", NULL, &response); + CHECK(cJSON_IsFalse(cJSON_GetObjectItem(response, "webrtcAudio")), + "audio is not advertised without WebRTC"); + cJSON_Delete(response); + + sinricpro_camera_controller_set_webrtc_offer_callback(controller, offer_callback, NULL); + run(controller, "getCameraCapabilities", NULL, &response); + CHECK(cJSON_IsTrue(cJSON_GetObjectItem(response, "webrtc")) && + cJSON_IsTrue(cJSON_GetObjectItem(response, "webrtcAudio")), + "a registered callback and audio are both reported"); + cJSON_Delete(response); + + sinricpro_camera_controller_destroy(controller); +} + +static void test_offer_and_ice_servers(void) +{ + printf("getWebRTCAnswer\n"); + + sinricpro_camera_controller_handle_t controller = sinricpro_camera_controller_create(); + sinricpro_camera_controller_set_webrtc_offer_callback(controller, offer_callback, NULL); + + const char *offer_sdp = "v=0\r\no=- 1 2 IN IP4 127.0.0.1\r\nm=application 9 UDP/DTLS/SCTP webrtc-datachannel\r\n"; + const char *answer_sdp = "v=0\r\no=- 3 4 IN IP4 192.168.1.20\r\na=candidate:1 1 udp 2130706431 192.168.1.20 50000 typ host\r\n"; + char *offer_b64 = encode(offer_sdp); + + /* One TURN entry with three URLs, a bare STUN string, a junk entry, and an + * array holding a non-string: every shape the flattening has to survive. */ + char request[1024]; + snprintf(request, sizeof(request), + "{\"offer\":\"%s\",\"iceServers\":[" + "{\"urls\":[\"stun:turn.sinric.pro:3478\",\"turn:turn.sinric.pro:3478?transport=udp\"," + "\"turns:turn.sinric.pro:443?transport=tcp\"]," + "\"username\":\"1757725200:user\",\"credential\":\"c2VjcmV0\"}," + "{\"urls\":\"stun:stun.l.google.com:19302\"}," + "42," + "{\"urls\":[7,\"turn:backup.example:3478\"]}]}", + offer_b64); + + memset(&seen, 0, sizeof(seen)); + seen.answer = answer_sdp; + seen.result = true; + + cJSON *response = NULL; + bool handled = run(controller, "getWebRTCAnswer", request, &response); + + CHECK(handled, "a successful callback succeeds"); + CHECK(seen.calls == 1 && strcmp(seen.offer, offer_sdp) == 0, "the offer reaches the callback decoded"); + CHECK(seen.server_count == 5, "each URL becomes one entry, non-strings skipped (%zu)", seen.server_count); + CHECK(strcmp(seen.url[1], "turn:turn.sinric.pro:3478?transport=udp") == 0 && + strcmp(seen.username[1], "1757725200:user") == 0 && strcmp(seen.credential[1], "c2VjcmV0") == 0, + "URLs in one entry share its credentials"); + CHECK(strcmp(seen.url[3], "stun:stun.l.google.com:19302") == 0 && + seen.username[3][0] == '\0' && seen.credential[3][0] == '\0', + "a string \"urls\" is accepted, with empty credentials when none are sent"); + CHECK(strcmp(seen.url[4], "turn:backup.example:3478") == 0, "the string after a non-string is kept"); + + char *expected = encode(answer_sdp); + cJSON *answer = cJSON_GetObjectItem(response, "answer"); + CHECK(cJSON_IsString(answer) && strcmp(answer->valuestring, expected) == 0, + "the answer is returned base64-encoded"); + free(expected); + cJSON_Delete(response); + + seen.result = false; + handled = run(controller, "getWebRTCAnswer", request, &response); + CHECK(!handled && cJSON_GetObjectItem(response, "answer") == NULL, + "a failing callback fails and returns no answer"); + cJSON_Delete(response); + + seen.result = true; + seen.answer = NULL; + handled = run(controller, "getWebRTCAnswer", request, &response); + CHECK(!handled, "success without an answer is reported as a failure"); + cJSON_Delete(response); + + free(offer_b64); + sinricpro_camera_controller_destroy(controller); +} + +static void test_rejected_requests(void) +{ + printf("rejected requests\n"); + + sinricpro_camera_controller_handle_t controller = sinricpro_camera_controller_create(); + cJSON *response = NULL; + + CHECK(!run(controller, "getWebRTCAnswer", "{\"offer\":\"djA9\"}", &response), + "an offer without a registered callback is not handled"); + cJSON_Delete(response); + + sinricpro_camera_controller_set_webrtc_offer_callback(controller, offer_callback, NULL); + memset(&seen, 0, sizeof(seen)); + + CHECK(!run(controller, "getWebRTCAnswer", "{\"offer\":\"@@@@\"}", &response) && seen.calls == 0, + "an offer that is not base64 never reaches the callback"); + cJSON_Delete(response); + + CHECK(!run(controller, "getWebRTCAnswer", "{\"iceServers\":[]}", &response) && seen.calls == 0, + "a request without an offer never reaches the callback"); + cJSON_Delete(response); + + CHECK(!run(controller, "setPowerState", "{\"state\":\"On\"}", &response), + "other actions are left to other capabilities"); + cJSON_Delete(response); + + CHECK(sinricpro_camera_controller_owns_action("getWebRTCAnswer") && + sinricpro_camera_controller_owns_action("getCameraCapabilities") && + !sinricpro_camera_controller_owns_action("setPowerState") && + !sinricpro_camera_controller_owns_action(NULL), + "action ownership"); + + sinricpro_camera_controller_destroy(controller); +} + +int main(void) +{ + test_capabilities(); + test_offer_and_ice_servers(); + test_rejected_requests(); + + printf("\n%s (%d failure%s)\n", failures ? "FAILED" : "PASSED", failures, failures == 1 ? "" : "s"); + return failures ? 1 : 0; +} diff --git a/test/host/test_frame_assembler.c b/test/host/test_frame_assembler.c new file mode 100644 index 0000000..ace5508 --- /dev/null +++ b/test/host/test_frame_assembler.c @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2019-2025 Sinric. All rights reserved. + * Licensed under Creative Commons Attribution-Share Alike (CC BY-SA) + * + * Host tests for websocket message reassembly. esp_websocket_client posts a + * frame larger than its 2 KB buffer as several data events; before reassembly + * each piece reached the JSON parser on its own, so any message above 2 KB + * (a camera WebRTC offer is 3-6 KB) was silently lost. + * + * Build and run: test/host/run.sh + */ + +#include "sinricpro_frame_assembler.h" + +#include +#include +#include + +static int failures; + +#define CHECK(cond, ...) \ + do { \ + if (cond) { \ + printf(" ok " __VA_ARGS__); \ + printf("\n"); \ + } else { \ + failures++; \ + printf(" FAIL " __VA_ARGS__); \ + printf(" (%s:%d)\n", __FILE__, __LINE__); \ + } \ + } while (0) + +/* Feeds a message the way the websocket client posts it: chunk-sized events, + * each with the total length and its own offset. */ +static sinricpro_frame_result_t feed_in_chunks(sinricpro_frame_assembler_t *assembler, + const char *text, size_t chunk, + const char **message, size_t *message_len, + int *pending_events) +{ + size_t total = strlen(text); + sinricpro_frame_result_t result = SINRICPRO_FRAME_PENDING; + *pending_events = 0; + + for (size_t offset = 0; offset < total; offset += chunk) { + size_t len = total - offset < chunk ? total - offset : chunk; + result = sinricpro_frame_assembler_feed(assembler, text + offset, len, offset, total, + message, message_len); + if (result == SINRICPRO_FRAME_PENDING) { + (*pending_events)++; + } + } + return result; +} + +static char *make_message(size_t length) +{ + char *text = malloc(length + 1); + for (size_t i = 0; i < length; i++) { + text[i] = (char)('a' + i % 26); + } + text[length] = '\0'; + return text; +} + +static void test_single_chunk(void) +{ + printf("single chunk\n"); + + sinricpro_frame_assembler_t assembler; + sinricpro_frame_assembler_init(&assembler, 16384); + + const char *message = NULL; + size_t length = 0; + const char *json = "{\"timestamp\":1757721600}"; + sinricpro_frame_result_t result = sinricpro_frame_assembler_feed( + &assembler, json, strlen(json), 0, strlen(json), &message, &length); + + CHECK(result == SINRICPRO_FRAME_COMPLETE, "a message within one event completes immediately"); + CHECK(length == strlen(json) && strcmp(message, json) == 0, "bytes and length are preserved"); + + result = sinricpro_frame_assembler_feed(&assembler, json, strlen(json), 0, 0, &message, &length); + CHECK(result == SINRICPRO_FRAME_COMPLETE, "a zero total length means the chunk is the whole message"); + + sinricpro_frame_assembler_free(&assembler); +} + +static void test_offer_sized_message(void) +{ + printf("offer-sized message in 2 KB chunks\n"); + + sinricpro_frame_assembler_t assembler; + sinricpro_frame_assembler_init(&assembler, 16384); + + char *text = make_message(5500); + const char *message = NULL; + size_t length = 0; + int pending = 0; + sinricpro_frame_result_t result = feed_in_chunks(&assembler, text, 2048, &message, &length, &pending); + + CHECK(pending == 2, "the first two of three events are held back (%d)", pending); + CHECK(result == SINRICPRO_FRAME_COMPLETE, "the last event completes the message"); + CHECK(length == 5500 && memcmp(message, text, 5500) == 0 && message[5500] == '\0', + "reassembled bytes match and are terminated"); + + free(text); + sinricpro_frame_assembler_free(&assembler); +} + +static void test_oversized_message(void) +{ + printf("message above the limit\n"); + + sinricpro_frame_assembler_t assembler; + sinricpro_frame_assembler_init(&assembler, 4096); + + char *big = make_message(9000); + const char *message = NULL; + size_t length = 0; + int pending = 0; + sinricpro_frame_result_t result = feed_in_chunks(&assembler, big, 2048, &message, &length, &pending); + + CHECK(result == SINRICPRO_FRAME_DROPPED, "dropped rather than parsed in pieces"); + CHECK(pending == 4, "reported once, on the last chunk, not per chunk (%d pending)", pending); + + const char *json = "{\"ok\":true}"; + result = sinricpro_frame_assembler_feed(&assembler, json, strlen(json), 0, strlen(json), &message, &length); + CHECK(result == SINRICPRO_FRAME_COMPLETE && strcmp(message, json) == 0, + "the next message is unaffected"); + + free(big); + sinricpro_frame_assembler_free(&assembler); +} + +static void test_out_of_sequence(void) +{ + printf("chunks out of sequence\n"); + + sinricpro_frame_assembler_t assembler; + sinricpro_frame_assembler_init(&assembler, 16384); + + const char *message = NULL; + size_t length = 0; + char *text = make_message(3000); + + sinricpro_frame_result_t result = sinricpro_frame_assembler_feed( + &assembler, text + 2048, 952, 2048, 3000, &message, &length); + CHECK(result == SINRICPRO_FRAME_DROPPED, "a continuation with no message in progress is dropped"); + + sinricpro_frame_assembler_feed(&assembler, text, 2048, 0, 3000, &message, &length); + result = sinricpro_frame_assembler_feed(&assembler, text + 2048, 952, 2048, 3100, &message, &length); + CHECK(result == SINRICPRO_FRAME_DROPPED, "a continuation claiming a different total is dropped"); + + sinricpro_frame_assembler_feed(&assembler, text, 2048, 0, 3000, &message, &length); + result = sinricpro_frame_assembler_feed(&assembler, text + 2048, 1000, 2048, 3000, &message, &length); + CHECK(result == SINRICPRO_FRAME_DROPPED, "a chunk running past the total is dropped"); + + sinricpro_frame_assembler_feed(&assembler, text, 2048, 0, 3000, &message, &length); + result = sinricpro_frame_assembler_feed(&assembler, text, 2048, 0, 3000, &message, &length); + CHECK(result == SINRICPRO_FRAME_PENDING, "a new message abandons an unfinished one"); + result = sinricpro_frame_assembler_feed(&assembler, text + 2048, 952, 2048, 3000, &message, &length); + CHECK(result == SINRICPRO_FRAME_COMPLETE && memcmp(message, text, 3000) == 0, + "and then completes normally"); + + free(text); + sinricpro_frame_assembler_free(&assembler); +} + +int main(void) +{ + test_single_chunk(); + test_offer_sized_message(); + test_oversized_message(); + test_out_of_sequence(); + + printf("\n%s (%d failure%s)\n", failures ? "FAILED" : "PASSED", failures, failures == 1 ? "" : "s"); + return failures ? 1 : 0; +} From faf3dd128e7952d77ddbd43d3705451fdff95844 Mon Sep 17 00:00:00 2001 From: Aruna Tennakoon Date: Tue, 15 Sep 2026 20:31:53 +0700 Subject: [PATCH 2/3] fix(camera): enable the esp-tls server API that esp_peer needs on ESP-IDF 5.x esp_peer compiles its TLS server transport unconditionally, but ESP-IDF 5.x declares esp_tls_cfg_server_t and esp_tls_server_session_create only when CONFIG_ESP_TLS_SERVER is set, so the example failed to compile on 5.x. ESP-IDF 6.x declares the API unconditionally and ignores the option. --- examples/camera/sdkconfig.defaults | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/camera/sdkconfig.defaults b/examples/camera/sdkconfig.defaults index e6064ad..05932cd 100644 --- a/examples/camera/sdkconfig.defaults +++ b/examples/camera/sdkconfig.defaults @@ -14,6 +14,10 @@ CONFIG_MBEDTLS_SSL_PROTO_DTLS=y CONFIG_MBEDTLS_SSL_DTLS_SRTP=y CONFIG_MBEDTLS_X509_CREATE_C=y +# esp_peer always compiles its TLS server transport. On ESP-IDF 5.x that API is +# declared only with this option; 6.x declares it unconditionally and ignores the line. +CONFIG_ESP_TLS_SERVER=y + # esp_peer and the camera driver do not fit the default 1 MB app partition. CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y CONFIG_PARTITION_TABLE_CUSTOM=y From 6cf7fefae16e1f72ea8dc94e8af5982c62979c23 Mon Sep 17 00:00:00 2001 From: Aruna Tennakoon Date: Tue, 15 Sep 2026 20:41:10 +0700 Subject: [PATCH 3/3] fix(camera): require ESP-IDF 5.5, the oldest release esp_peer links against esp_peer ships a prebuilt libpeer_default.a that calls esp_log(), which ESP-IDF added in 5.5, so the camera example cannot link on older releases. The example previously claimed 5.1 or later and CI built it on the 5.1 image. - Build the camera example in CI on ESP-IDF v5.5, the floor, and v6.1. - State the 5.5 minimum in both component manifests, the example README, the README and the CHANGELOG. - Drop CONFIG_ESP_TLS_SERVER: from 5.3 the esp-tls server API is always declared, so the option only helped releases esp_peer cannot link on. - Drop the webrtc_camera fallback to the pre-5.3 driver component. --- .github/workflows/build-test.yml | 9 ++++++--- CHANGELOG.md | 4 +++- README.md | 2 +- examples/camera/README.md | 2 +- .../camera/components/webrtc_camera/CMakeLists.txt | 10 +--------- .../camera/components/webrtc_camera/idf_component.yml | 3 ++- examples/camera/main/idf_component.yml | 2 +- examples/camera/sdkconfig.defaults | 4 ---- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e4d5986..6f9d2dc 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -58,14 +58,17 @@ jobs: idf.py size # Camera live view. The example's board profiles cover ESP32 and ESP32-S3 only, - # so it cannot join the main matrix's S2 and C3 targets. + # so it cannot join the main matrix's S2 and C3 targets. esp_peer ships a + # prebuilt library that links only against ESP-IDF 5.5 or later, so 5.5 is the + # floor this job proves, alongside the current release. build-camera: runs-on: ubuntu-latest container: - image: espressif/idf:v5.1 + image: espressif/idf:${{ matrix.idf-version }} strategy: fail-fast: false matrix: + idf-version: [v5.5, v6.1] idf-target: [esp32, esp32s3] steps: @@ -78,7 +81,7 @@ jobs: run: | find . -type d -name "managed_components" -exec rm -rf {} + || true - - name: Build camera for ${{ matrix.idf-target }} + - name: Build camera for ${{ matrix.idf-target }} on ${{ matrix.idf-version }} working-directory: examples/camera run: | . $IDF_PATH/export.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index abd1ce7..0c8ab15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,9 @@ - feat: `examples/camera` streams JPEG frames over a WebRTC DataChannel through a `webrtc_camera` component built on `esp_peer`, with resolution, frame rate, flash, flip and mirror controls, automatic quality, and the XIAO ESP32S3 Sense - microphone. The SinricPro component itself gains no dependencies. + microphone. The SinricPro component itself gains no dependencies. The example + needs ESP-IDF 5.5 or later, because `esp_peer`'s prebuilt library links only + against 5.5 and newer. - feat: `sinricpro_set_response_message()`, so a callback can tell the client why a request failed. - feat: Kconfig `SINRICPRO_MAX_MESSAGE_SIZE` (default 16 KB). diff --git a/README.md b/README.md index 3432448..ac1d5de 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ All devices below have complete API support and working examples: - ✅ **Speaker** - Volume, mute, media control, equalizer, modes ### Cameras -- ✅ **Camera** - WebRTC live view in the SinricPro portal and app, with remote resolution, frame rate and flash control +- ✅ **Camera** - WebRTC live view in the SinricPro portal and app, with remote resolution, frame rate and flash control (example needs ESP-IDF 5.5+) ### Additional Devices (API Only) - ✅ Air Quality Sensor - PM1, PM2.5, PM10 measurements diff --git a/examples/camera/README.md b/examples/camera/README.md index df5672f..c32df5a 100644 --- a/examples/camera/README.md +++ b/examples/camera/README.md @@ -4,7 +4,7 @@ Streams an ESP32 or ESP32-S3 camera to the SinricPro portal and app, from anywhe ## Requirements -- ESP-IDF 5.1 or later +- ESP-IDF 5.5 or later. `esp_peer` includes a prebuilt library that links only against 5.5 and newer. - An ESP32 or ESP32-S3 camera board **with PSRAM** - 4 MB of flash or more - A Wi-Fi signal of **−75 dBm or better** at the board. Below about −80 dBm the Wi-Fi driver's transmit buffers stop recycling fast enough and the DTLS handshake cannot complete, even though free heap looks healthy. diff --git a/examples/camera/components/webrtc_camera/CMakeLists.txt b/examples/camera/components/webrtc_camera/CMakeLists.txt index aba21ca..c1a61e3 100644 --- a/examples/camera/components/webrtc_camera/CMakeLists.txt +++ b/examples/camera/components/webrtc_camera/CMakeLists.txt @@ -1,11 +1,3 @@ -# driver/gpio.h moved out of the monolithic driver component in ESP-IDF 5.3; -# from 6.0 that component no longer provides it at all. -if("${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}" VERSION_GREATER_EQUAL "5.3") - set(gpio_component esp_driver_gpio) -else() - set(gpio_component driver) -endif() - idf_component_register( SRCS "webrtc_camera.c" @@ -18,7 +10,7 @@ idf_component_register( esp32-camera PRIV_REQUIRES cjson + esp_driver_gpio esp_timer esp_wifi - ${gpio_component} ) diff --git a/examples/camera/components/webrtc_camera/idf_component.yml b/examples/camera/components/webrtc_camera/idf_component.yml index 3b1c7d7..498ce00 100644 --- a/examples/camera/components/webrtc_camera/idf_component.yml +++ b/examples/camera/components/webrtc_camera/idf_component.yml @@ -1,7 +1,8 @@ description: "SinricPro WebRTC live view for ESP32 cameras: JPEG frames over a DataChannel" dependencies: + # esp_peer's prebuilt library calls esp_log(), which ESP-IDF added in 5.5. idf: - version: ">=5.1" + version: ">=5.5" espressif/esp_peer: version: "^1.5.5" espressif/esp32-camera: diff --git a/examples/camera/main/idf_component.yml b/examples/camera/main/idf_component.yml index be8aaca..f775e43 100644 --- a/examples/camera/main/idf_component.yml +++ b/examples/camera/main/idf_component.yml @@ -1,6 +1,6 @@ dependencies: idf: - version: ">=5.1" + version: ">=5.5" espressif/esp_websocket_client: version: "^1.2.0" espressif/cjson: diff --git a/examples/camera/sdkconfig.defaults b/examples/camera/sdkconfig.defaults index 05932cd..e6064ad 100644 --- a/examples/camera/sdkconfig.defaults +++ b/examples/camera/sdkconfig.defaults @@ -14,10 +14,6 @@ CONFIG_MBEDTLS_SSL_PROTO_DTLS=y CONFIG_MBEDTLS_SSL_DTLS_SRTP=y CONFIG_MBEDTLS_X509_CREATE_C=y -# esp_peer always compiles its TLS server transport. On ESP-IDF 5.x that API is -# declared only with this option; 6.x declares it unconditionally and ignores the line. -CONFIG_ESP_TLS_SERVER=y - # esp_peer and the camera driver do not fit the default 1 MB app partition. CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y CONFIG_PARTITION_TABLE_CUSTOM=y