From b2427c0c52b4881cc30d47f336034b7c82ecd790 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 1 Jul 2026 14:56:48 -0600 Subject: [PATCH 1/2] Initial user_data example --- CMakeLists.txt | 1 + README.md | 10 ++ user_data/CMakeLists.txt | 27 +++ user_data/README.md | 34 ++++ user_data/common/cli_utils.h | 102 +++++++++++ user_data/common/constants.h | 27 +++ user_data/common/json_converters.cpp | 48 +++++ user_data/common/json_converters.h | 28 +++ user_data/common/messages.h | 29 +++ user_data/consumer/CMakeLists.txt | 21 +++ user_data/consumer/main.cpp | 220 +++++++++++++++++++++++ user_data/producer/CMakeLists.txt | 21 +++ user_data/producer/main.cpp | 181 +++++++++++++++++++ user_timestamped_video/README.md | 25 +-- user_timestamped_video/consumer/main.cpp | 15 +- user_timestamped_video/producer/main.cpp | 14 +- 16 files changed, 785 insertions(+), 18 deletions(-) create mode 100644 user_data/CMakeLists.txt create mode 100644 user_data/README.md create mode 100644 user_data/common/cli_utils.h create mode 100644 user_data/common/constants.h create mode 100644 user_data/common/json_converters.cpp create mode 100644 user_data/common/json_converters.h create mode 100644 user_data/common/messages.h create mode 100644 user_data/consumer/CMakeLists.txt create mode 100644 user_data/consumer/main.cpp create mode 100644 user_data/producer/CMakeLists.txt create mode 100644 user_data/producer/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 83b916a..9b54342 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,3 +93,4 @@ add_subdirectory(platform_audio) add_subdirectory(ping_pong/ping ping_pong_ping) add_subdirectory(ping_pong/pong ping_pong_pong) add_subdirectory(user_timestamped_video) +add_subdirectory(user_data) diff --git a/README.md b/README.md index 88107bd..a97340a 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,16 @@ WebRTC's platform Audio Device Module: ./build/platform_audio/sender/PlatformAudioSender ``` +### UserData + +The `user_data` example shows how to attach small sensor readings to video +frames using SDK `v1.3.0` frame metadata: + +```bash +./build/user_data/producer/UserDataProducer +./build/user_data/consumer/UserDataConsumer +``` + ### Supported platforms Prebuilt SDKs are downloaded automatically for: diff --git a/user_data/CMakeLists.txt b/user_data/CMakeLists.txt new file mode 100644 index 0000000..8739984 --- /dev/null +++ b/user_data/CMakeLists.txt @@ -0,0 +1,27 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library(user_data_support STATIC + common/json_converters.cpp + common/json_converters.h + common/cli_utils.h + common/constants.h + common/messages.h +) + +target_include_directories(user_data_support PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/common) +target_link_libraries(user_data_support PUBLIC nlohmann_json::nlohmann_json) + +add_subdirectory(producer) +add_subdirectory(consumer) diff --git a/user_data/README.md b/user_data/README.md new file mode 100644 index 0000000..8804b91 --- /dev/null +++ b/user_data/README.md @@ -0,0 +1,34 @@ +# UserData + +This example shows how to pair small application data with video frames: + +- `UserDataProducer` publishes a synthetic video track named `"sensor-camera"`. + Each frame carries `VideoCaptureOptions::metadata.frame_id` and + `VideoCaptureOptions::metadata.user_timestamp_us`. +- The producer also attaches a small JSON temperature reading to + `VideoCaptureOptions::metadata.user_data`. +- `UserDataConsumer` subscribes to the video track and parses the inline + `user_data` payload on each received frame. + +Run them in the same room with different participant identities: + +```sh +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserDataProducer +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserDataConsumer +``` + +Requirements: + +- LiveKit C++ SDK `v1.3.0` or newer. This example uses packet-trailer frame IDs + and user data in `VideoFrameMetadata`. +- To pin the SDK version when configuring the examples, pass + `-DLIVEKIT_SDK_VERSION=1.3.0` to CMake. + +The data payload is intentionally small: + +```json +{"frame_id":42,"timestamp_us":1782930000000000,"temperature_c":23.4} +``` + +In a robotics or sensor-fusion application, this lets a small telemetry value +travel with the frame it describes without requiring a companion data track. diff --git a/user_data/common/cli_utils.h b/user_data/common/cli_utils.h new file mode 100644 index 0000000..cbd0e7d --- /dev/null +++ b/user_data/common/cli_utils.h @@ -0,0 +1,102 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace user_data { + +enum class ParseResult { Ok, Help, Error }; + +struct CliOptions { + std::string url; + std::string token; +}; + +inline std::atomic g_running{true}; + +inline void handleSignal(int) { g_running.store(false); } + +inline bool isRunning() { return g_running.load(std::memory_order_relaxed); } + +inline void installSignalHandlers() { + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif +} + +inline std::string getenvOrEmpty(const char* name) { + const char* value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +inline void printUsage(const char* program) { + std::cerr << "Usage:\n" + << " " << program << " \n" + << "or:\n" + << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program << "\n"; +} + +inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) { + std::vector positional; + options = CliOptions{}; + + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + return ParseResult::Help; + } + if (!arg.empty() && arg[0] == '-') { + return ParseResult::Error; + } + + positional.push_back(arg); + } + + if (positional.size() > 2) { + return ParseResult::Error; + } + + options.url = getenvOrEmpty("LIVEKIT_URL"); + options.token = getenvOrEmpty("LIVEKIT_TOKEN"); + + if (positional.size() == 2) { + options.url = positional[0]; + options.token = positional[1]; + } else if (positional.size() == 1) { + return ParseResult::Error; + } + + return (options.url.empty() || options.token.empty()) ? ParseResult::Error : ParseResult::Ok; +} + +inline std::vector toPayload(const std::string& text) { + return std::vector(text.begin(), text.end()); +} + +inline std::string toString(const std::vector& payload) { + return std::string(payload.begin(), payload.end()); +} + +} // namespace user_data diff --git a/user_data/common/constants.h b/user_data/common/constants.h new file mode 100644 index 0000000..1482884 --- /dev/null +++ b/user_data/common/constants.h @@ -0,0 +1,27 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace user_data { + +inline constexpr char kVideoTrackName[] = "sensor-camera"; + +inline constexpr char kFrameIdKey[] = "frame_id"; +inline constexpr char kTimestampUsKey[] = "timestamp_us"; +inline constexpr char kTemperatureCKey[] = "temperature_c"; + +} // namespace user_data diff --git a/user_data/common/json_converters.cpp b/user_data/common/json_converters.cpp new file mode 100644 index 0000000..f02a38a --- /dev/null +++ b/user_data/common/json_converters.cpp @@ -0,0 +1,48 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "json_converters.h" + +#include +#include + +#include "constants.h" + +namespace user_data { + +std::string sensorReadingToJson(const SensorReading& reading) { + nlohmann::json json; + json[kFrameIdKey] = reading.frame_id; + json[kTimestampUsKey] = reading.timestamp_us; + json[kTemperatureCKey] = reading.temperature_c; + return json.dump(); +} + +SensorReading sensorReadingFromJson(const std::string& json_text) { + try { + const auto json = nlohmann::json::parse(json_text); + + SensorReading reading; + reading.frame_id = json.at(kFrameIdKey).get(); + reading.timestamp_us = json.at(kTimestampUsKey).get(); + reading.temperature_c = json.at(kTemperatureCKey).get(); + return reading; + } catch (const nlohmann::json::exception& error) { + throw std::runtime_error(std::string("Failed to parse sensor reading JSON: ") + error.what()); + } +} + +} // namespace user_data diff --git a/user_data/common/json_converters.h b/user_data/common/json_converters.h new file mode 100644 index 0000000..72ff49a --- /dev/null +++ b/user_data/common/json_converters.h @@ -0,0 +1,28 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "messages.h" + +namespace user_data { + +std::string sensorReadingToJson(const SensorReading& reading); +SensorReading sensorReadingFromJson(const std::string& json_text); + +} // namespace user_data diff --git a/user_data/common/messages.h b/user_data/common/messages.h new file mode 100644 index 0000000..fe87602 --- /dev/null +++ b/user_data/common/messages.h @@ -0,0 +1,29 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace user_data { + +struct SensorReading { + std::uint32_t frame_id = 0; + std::uint64_t timestamp_us = 0; + double temperature_c = 0.0; +}; + +} // namespace user_data diff --git a/user_data/consumer/CMakeLists.txt b/user_data/consumer/CMakeLists.txt new file mode 100644 index 0000000..f2b0250 --- /dev/null +++ b/user_data/consumer/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(UserDataConsumer + main.cpp +) + +target_link_libraries(UserDataConsumer PRIVATE user_data_support ${LIVEKIT_CORE_TARGET}) + +livekit_copy_windows_runtime_dlls(UserDataConsumer) diff --git a/user_data/consumer/main.cpp b/user_data/consumer/main.cpp new file mode 100644 index 0000000..c9eeef5 --- /dev/null +++ b/user_data/consumer/main.cpp @@ -0,0 +1,220 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// UserDataConsumer +/// +/// Receives video frame events with v1.3.0 metadata and parses small +/// temperature readings from `VideoFrameMetadata::user_data`. +/// +/// Usage: +/// UserDataConsumer +/// +/// Or via environment variables: +/// LIVEKIT_URL, LIVEKIT_TOKEN + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cli_utils.h" +#include "constants.h" +#include "json_converters.h" +#include "livekit/livekit.h" +#include "messages.h" + +using namespace livekit; + +namespace { + +std::optional metadataFrameId(const std::optional& metadata) { + if (!metadata || !metadata->frame_id.has_value()) { + return std::nullopt; + } + + return *metadata->frame_id; +} + +std::optional metadataSensorReading(const std::optional& metadata) { + if (!metadata || !metadata->user_data.has_value()) { + return std::nullopt; + } + + return user_data::sensorReadingFromJson(user_data::toString(*metadata->user_data)); +} + +std::string formatOptionalTimestamp(std::optional timestamp_us) { + return timestamp_us ? std::to_string(*timestamp_us) : std::string("n/a"); +} + +class UserDataConsumerDelegate : public RoomDelegate { +public: + explicit UserDataConsumerDelegate(Room& room) : room_(room) {} + + void registerExistingParticipants() { + for (const auto& weak_participant : room_.remoteParticipants()) { + if (auto participant = weak_participant.lock()) { + registerRemoteVideoCallback(participant->identity()); + } else { + throw std::runtime_error("unable to lock provided remote participant"); + } + } + } + + void clearAllCallbacks() { + std::unordered_set identities; + { + std::lock_guard lock(mutex_); + identities.swap(registered_identities_); + } + + for (const auto& identity : identities) { + room_.clearOnVideoFrameCallback(identity, std::string(user_data::kVideoTrackName)); + } + } + + void onParticipantConnected(Room&, const ParticipantConnectedEvent& event) override { + if (!event.participant) { + return; + } + + std::cout << "[consumer] participant connected: " << event.participant->identity() << "\n"; + registerRemoteVideoCallback(event.participant->identity()); + } + + void onParticipantDisconnected(Room&, const ParticipantDisconnectedEvent& event) override { + if (!event.participant) { + return; + } + + const std::string identity = event.participant->identity(); + room_.clearOnVideoFrameCallback(identity, std::string(user_data::kVideoTrackName)); + { + std::lock_guard lock(mutex_); + registered_identities_.erase(identity); + } + + std::cout << "[consumer] participant disconnected: " << identity << "\n"; + } + +private: + void registerRemoteVideoCallback(const std::string& identity) { + { + std::lock_guard lock(mutex_); + if (!registered_identities_.insert(identity).second) { + return; + } + } + + VideoStream::Options stream_options; + stream_options.format = VideoBufferType::RGBA; + + room_.setOnVideoFrameEventCallback( + identity, std::string(user_data::kVideoTrackName), + [identity](const VideoFrameEvent& event) { + try { + const auto frame_id = metadataFrameId(event.metadata); + const auto reading = metadataSensorReading(event.metadata); + + std::cout << "[consumer] from=" << identity << " size=" << event.frame.width() << "x" + << event.frame.height() << " capture_ts_us=" << event.timestamp_us + << " metadata_frame_id=" << (frame_id ? std::to_string(*frame_id) : std::string("n/a")) + << " user_ts_us=" << formatOptionalTimestamp(event.metadata ? event.metadata->user_timestamp_us + : std::nullopt); + + if (reading) { + std::cout << std::fixed << std::setprecision(2) << " reading_frame_id=" << reading->frame_id + << " temperature_c=" << reading->temperature_c + << " reading_ts_us=" << reading->timestamp_us + << " user_data_bytes=" << event.metadata->user_data->size(); + } else { + std::cout << " user_data=n/a"; + } + + std::cout << "\n"; + } catch (const std::exception& error) { + std::cerr << "[consumer] failed to process frame metadata from " << identity << ": " << error.what() + << "\n"; + } + }, + stream_options); + + std::cout << "[consumer] listening for video frames from " << identity << " track=\"" + << user_data::kVideoTrackName << "\"\n"; + } + + Room& room_; + std::mutex mutex_; + std::unordered_set registered_identities_; +}; + +} // namespace + +int main(int argc, char* argv[]) { + user_data::CliOptions cli_options; + + const user_data::ParseResult parse_result = user_data::parseArgs(argc, argv, cli_options); + if (parse_result != user_data::ParseResult::Ok) { + user_data::printUsage(argv[0]); + return parse_result == user_data::ParseResult::Help ? 0 : 1; + } + + user_data::installSignalHandlers(); + + livekit::initialize(livekit::LogLevel::Info); + int exit_code = 0; + + { + Room room; + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + UserDataConsumerDelegate delegate(room); + room.setDelegate(&delegate); + + std::cout << "[consumer] connecting to " << cli_options.url << "\n"; + if (!room.connect(cli_options.url, cli_options.token, options)) { + std::cerr << "[consumer] failed to connect\n"; + exit_code = 1; + } else { + if (auto lp = room.localParticipant().lock()) { + std::cout << "[consumer] connected as " << lp->identity() << " to room '" << room.roomInfo().name << "'\n"; + } else { + throw std::runtime_error("unable to lock local participant"); + } + + delegate.registerExistingParticipants(); + + while (user_data::isRunning()) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + delegate.clearAllCallbacks(); + } + + room.setDelegate(nullptr); + } + + livekit::shutdown(); + return exit_code; +} diff --git a/user_data/producer/CMakeLists.txt b/user_data/producer/CMakeLists.txt new file mode 100644 index 0000000..ece31de --- /dev/null +++ b/user_data/producer/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(UserDataProducer + main.cpp +) + +target_link_libraries(UserDataProducer PRIVATE user_data_support ${LIVEKIT_CORE_TARGET}) + +livekit_copy_windows_runtime_dlls(UserDataProducer) diff --git a/user_data/producer/main.cpp b/user_data/producer/main.cpp new file mode 100644 index 0000000..944d319 --- /dev/null +++ b/user_data/producer/main.cpp @@ -0,0 +1,181 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// UserDataProducer +/// +/// Publishes a synthetic camera track. Each video frame carries v1.3.0 frame +/// metadata: `frame_id`, `user_timestamp_us`, and a compact temperature reading +/// in `user_data`. +/// +/// Usage: +/// UserDataProducer +/// +/// Or via environment variables: +/// LIVEKIT_URL, LIVEKIT_TOKEN + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cli_utils.h" +#include "constants.h" +#include "json_converters.h" +#include "livekit/livekit.h" +#include "messages.h" + +using namespace livekit; + +namespace { + +constexpr int kFrameWidth = 640; +constexpr int kFrameHeight = 360; +constexpr int kFrameIntervalMs = 200; + +std::uint64_t nowEpochUs() { + return static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count()); +} + +double simulatedTemperatureC(std::uint32_t frame_id) { + return 22.0 + (2.5 * std::sin(static_cast(frame_id) * 0.15)); +} + +void fillFrame(VideoFrame& frame, double temperature_c) { + const double normalized = std::max(0.0, std::min(1.0, (temperature_c - 18.0) / 10.0)); + const auto red = static_cast(80.0 + (normalized * 175.0)); + const auto blue = static_cast(255.0 - (normalized * 120.0)); + const auto green = static_cast(90); + + std::uint8_t* data = frame.data(); + for (std::size_t i = 0; i < frame.dataSize(); i += 4) { + data[i + 0] = blue; + data[i + 1] = green; + data[i + 2] = red; + data[i + 3] = 255; + } +} + +} // namespace + +int main(int argc, char* argv[]) { + user_data::CliOptions cli_options; + + const user_data::ParseResult parse_result = user_data::parseArgs(argc, argv, cli_options); + if (parse_result != user_data::ParseResult::Ok) { + user_data::printUsage(argv[0]); + return parse_result == user_data::ParseResult::Help ? 0 : 1; + } + + user_data::installSignalHandlers(); + + livekit::initialize(livekit::LogLevel::Info); + int exit_code = 0; + + { + Room room; + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + std::cout << "[producer] connecting to " << cli_options.url << "\n"; + if (!room.connect(cli_options.url, cli_options.token, options)) { + std::cerr << "[producer] failed to connect\n"; + exit_code = 1; + } else { + std::shared_ptr video_track; + auto source = std::make_shared(kFrameWidth, kFrameHeight); + + try { + { + auto lp = room.localParticipant().lock(); + if (!lp) throw std::runtime_error("local participant unavailable"); + + std::cout << "[producer] connected as " << lp->identity() << " to room '" << room.roomInfo().name << "'\n"; + + video_track = LocalVideoTrack::createLocalVideoTrack(user_data::kVideoTrackName, source); + + TrackPublishOptions publish_options; + publish_options.source = TrackSource::SOURCE_CAMERA; + publish_options.frame_metadata_features = FrameMetadataFeatures{}; + publish_options.frame_metadata_features->user_timestamp = true; + publish_options.frame_metadata_features->frame_id = true; + publish_options.frame_metadata_features->user_data = true; + + lp->publishTrack(video_track, publish_options); + std::cout << "[producer] published video track \"" << user_data::kVideoTrackName + << "\" with timestamp, frame id, and user data metadata\n"; + } + + VideoFrame frame = VideoFrame::create(kFrameWidth, kFrameHeight, VideoBufferType::BGRA); + const auto capture_start = std::chrono::steady_clock::now(); + std::uint32_t frame_id = 0; + auto next_frame_at = std::chrono::steady_clock::now(); + + std::cout << "[producer] sending synthetic video with inline temperature metadata; Ctrl-C to exit\n"; + while (user_data::isRunning()) { + user_data::SensorReading reading; + reading.frame_id = frame_id; + reading.timestamp_us = nowEpochUs(); + reading.temperature_c = simulatedTemperatureC(frame_id); + + fillFrame(frame, reading.temperature_c); + + VideoCaptureOptions capture_options; + capture_options.timestamp_us = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - capture_start) + .count()); + capture_options.rotation = VideoRotation::VIDEO_ROTATION_0; + capture_options.metadata = VideoFrameMetadata{}; + capture_options.metadata->user_timestamp_us = reading.timestamp_us; + capture_options.metadata->frame_id = reading.frame_id; + capture_options.metadata->user_data = user_data::toPayload(user_data::sensorReadingToJson(reading)); + + source->captureFrame(frame, capture_options); + + if (frame_id % 5 == 0) { + std::cout << std::fixed << std::setprecision(2) << "[producer] frame_id=" << reading.frame_id + << " user_ts_us=" << reading.timestamp_us << " temperature_c=" << reading.temperature_c + << " user_data_bytes=" << capture_options.metadata->user_data->size() << "\n"; + } + + ++frame_id; + next_frame_at += std::chrono::milliseconds(kFrameIntervalMs); + std::this_thread::sleep_until(next_frame_at); + } + } catch (const std::exception& error) { + std::cerr << "[producer] error: " << error.what() << "\n"; + exit_code = 1; + } + + if (auto lp = room.localParticipant().lock()) { + if (video_track && video_track->publication()) { + lp->unpublishTrack(video_track->publication()->sid()); + } + } + } + } + + livekit::shutdown(); + return exit_code; +} diff --git a/user_timestamped_video/README.md b/user_timestamped_video/README.md index ea9fd27..3846aca 100644 --- a/user_timestamped_video/README.md +++ b/user_timestamped_video/README.md @@ -4,8 +4,9 @@ This example is split into two executables and can demonstrate all four producer/consumer combinations: - `UserTimestampedVideoProducer` publishes a synthetic video track named - `"timestamped-camera"` and stamps each frame with - `VideoCaptureOptions::metadata.user_timestamp_us`. + `"timestamped-camera"` and stamps each frame with both + `VideoCaptureOptions::metadata.user_timestamp_us` and + `VideoCaptureOptions::metadata.frame_id`. - `UserTimestampedVideoConsumer` subscribes to the remote `"timestamped-camera"` track by name with either the rich or legacy callback path. @@ -19,19 +20,19 @@ LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestamped Requirements: -- LiveKit C++ SDK `v0.3.4` or newer. This example uses - `VideoFrameMetadata` and `setOnVideoFrameEventCallback`, which are not - available in older SDK releases. +- LiveKit C++ SDK `v1.3.0` or newer. This example uses `VideoFrameMetadata` + frame IDs and `setOnVideoFrameEventCallback`, which are not available in + older SDK releases. - To pin the SDK version when configuring the examples, pass - `-DLIVEKIT_SDK_VERSION=0.3.4` to CMake. + `-DLIVEKIT_SDK_VERSION=1.3.0` to CMake. Flags: -- Producer default: sends user timestamps -- Producer `--with-user-timestamp`: explicitly sends user timestamps -- Producer `--without-user-timestamp`: does not send user timestamps -- Consumer default: reads user timestamps through `setOnVideoFrameEventCallback` -- Consumer `--with-user-timestamp`: explicitly reads user timestamps through +- Producer default: sends user timestamps and frame IDs +- Producer `--with-user-timestamp`: explicitly sends user timestamps and frame IDs +- Producer `--without-user-timestamp`: does not send frame metadata +- Consumer default: reads metadata through `setOnVideoFrameEventCallback` +- Consumer `--with-user-timestamp`: explicitly reads metadata through `setOnVideoFrameEventCallback` - Consumer `--without-user-timestamp`: ignores metadata through the legacy `setOnVideoFrameCallback` @@ -59,6 +60,8 @@ Matrix: Timestamp note: - `user_ts_us` is application metadata and is the value to compare end to end. +- `metadata_frame_id` is application metadata and should match the producer's + frame counter when the frame metadata path is enabled. - `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`. - `capture_ts_us` on the consumer is the received WebRTC frame timestamp. - Producer and consumer `capture_ts_us` values are not expected to match exactly, diff --git a/user_timestamped_video/consumer/main.cpp b/user_timestamped_video/consumer/main.cpp index 6ba5d32..f74dbef 100644 --- a/user_timestamped_video/consumer/main.cpp +++ b/user_timestamped_video/consumer/main.cpp @@ -17,8 +17,8 @@ /// UserTimestampedVideoConsumer /// /// Receives remote video frames via `Room::setOnVideoFrameEventCallback()` and -/// logs any `VideoFrameMetadata::user_timestamp_us` values that arrive. Pair -/// with `UserTimestampedVideoProducer` running in another process. +/// logs any `VideoFrameMetadata::user_timestamp_us` and `frame_id` values that +/// arrive. Pair with `UserTimestampedVideoProducer` running in another process. /// /// Usage: /// UserTimestampedVideoConsumer @@ -53,6 +53,14 @@ std::string formatUserTimestamp(const std::optional& metadat return std::to_string(*metadata->user_timestamp_us); } +std::string formatFrameId(const std::optional& metadata) { + if (!metadata || !metadata->frame_id.has_value()) { + return "n/a"; + } + + return std::to_string(*metadata->frame_id); +} + class UserTimestampedVideoConsumerDelegate : public RoomDelegate { public: UserTimestampedVideoConsumerDelegate(Room& room, bool read_user_timestamp) @@ -111,6 +119,7 @@ class UserTimestampedVideoConsumerDelegate : public RoomDelegate { [identity](const VideoFrameEvent& event) { std::cout << "[consumer] from=" << identity << " size=" << event.frame.width() << "x" << event.frame.height() << " capture_ts_us=" << event.timestamp_us + << " metadata_frame_id=" << formatFrameId(event.metadata) << " user_ts_us=" << formatUserTimestamp(event.metadata) << " rotation=" << static_cast(event.rotation) << "\n"; }, @@ -120,7 +129,7 @@ class UserTimestampedVideoConsumerDelegate : public RoomDelegate { identity, std::string(kTrackName), [identity](const VideoFrame& frame, const std::int64_t timestamp_us) { std::cout << "[consumer] from=" << identity << " size=" << frame.width() << "x" << frame.height() - << " capture_ts_us=" << timestamp_us << " user_ts_us=ignored\n"; + << " capture_ts_us=" << timestamp_us << " metadata_frame_id=ignored user_ts_us=ignored\n"; }, stream_options); } diff --git a/user_timestamped_video/producer/main.cpp b/user_timestamped_video/producer/main.cpp index c58a983..2c297e4 100644 --- a/user_timestamped_video/producer/main.cpp +++ b/user_timestamped_video/producer/main.cpp @@ -17,9 +17,9 @@ /// UserTimestampedVideoProducer /// /// Publishes a synthetic camera track and stamps each frame with -/// `VideoCaptureOptions::metadata.user_timestamp_us`. Pair with +/// `VideoCaptureOptions::metadata.user_timestamp_us` and `frame_id`. Pair with /// `UserTimestampedVideoConsumer` in another process to observe the user -/// timestamps flowing end to end. +/// metadata flowing end to end. /// /// Usage: /// UserTimestampedVideoProducer @@ -105,13 +105,15 @@ int main(int argc, char* argv[]) { try { TrackPublishOptions publish_options; publish_options.source = TrackSource::SOURCE_CAMERA; - publish_options.packet_trailer_features.user_timestamp = cli_options.use_user_timestamp; + publish_options.frame_metadata_features = FrameMetadataFeatures{}; + publish_options.frame_metadata_features->user_timestamp = cli_options.use_user_timestamp; + publish_options.frame_metadata_features->frame_id = cli_options.use_user_timestamp; { auto lp = room.localParticipant().lock(); if (!lp) throw std::runtime_error("local participant unavailable"); lp->publishTrack(track, publish_options); - std::cout << "[producer] published camera track with user timestamp " + std::cout << "[producer] published camera track with frame metadata " << (cli_options.use_user_timestamp ? "enabled" : "disabled") << "\n"; } @@ -133,12 +135,16 @@ int main(int argc, char* argv[]) { if (cli_options.use_user_timestamp) { capture_options.metadata = VideoFrameMetadata{}; capture_options.metadata->user_timestamp_us = nowEpochUs(); + capture_options.metadata->frame_id = frame_index; } source->captureFrame(frame, capture_options); if (frame_index % 5 == 0) { std::cout << "[producer] frame=" << frame_index << " capture_ts_us=" << capture_options.timestamp_us + << " metadata_frame_id=" + << (cli_options.use_user_timestamp ? std::to_string(*capture_options.metadata->frame_id) + : std::string("disabled")) << " user_ts_us=" << (cli_options.use_user_timestamp ? std::to_string(*capture_options.metadata->user_timestamp_us) : std::string("disabled")) From a5a1f525b461648f79102ae227b9f46d13a962a3 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 1 Jul 2026 20:27:15 -0600 Subject: [PATCH 2/2] Big overhaul into frame metadata, improved outputs, docs, etc. --- CMakeLists.txt | 3 +- README.md | 20 -- {user_data => frame_metadata}/CMakeLists.txt | 6 +- frame_metadata/README.md | 50 ++++ .../common/cli_utils.h | 24 +- .../common/constants.h | 6 +- .../common/json_converters.cpp | 8 +- .../common/json_converters.h | 4 +- .../common/messages.h | 8 +- .../consumer}/CMakeLists.txt | 6 +- .../consumer/main.cpp | 121 +++++----- .../producer}/CMakeLists.txt | 6 +- .../producer/main.cpp | 87 +++---- user_data/README.md | 34 --- user_data/consumer/main.cpp | 220 ------------------ user_timestamped_video/CMakeLists.txt | 16 -- user_timestamped_video/README.md | 69 ------ user_timestamped_video/common/cli_utils.h | 104 --------- .../consumer/CMakeLists.txt | 22 -- .../producer/CMakeLists.txt | 22 -- user_timestamped_video/producer/main.cpp | 174 -------------- 21 files changed, 191 insertions(+), 819 deletions(-) rename {user_data => frame_metadata}/CMakeLists.txt (77%) create mode 100644 frame_metadata/README.md rename {user_data => frame_metadata}/common/cli_utils.h (77%) rename {user_data => frame_metadata}/common/constants.h (82%) rename {user_data => frame_metadata}/common/json_converters.cpp (82%) rename {user_data => frame_metadata}/common/json_converters.h (93%) rename {user_data => frame_metadata}/common/messages.h (83%) rename {user_data/producer => frame_metadata/consumer}/CMakeLists.txt (74%) rename {user_timestamped_video => frame_metadata}/consumer/main.cpp (54%) rename {user_data/consumer => frame_metadata/producer}/CMakeLists.txt (74%) rename {user_data => frame_metadata}/producer/main.cpp (61%) delete mode 100644 user_data/README.md delete mode 100644 user_data/consumer/main.cpp delete mode 100644 user_timestamped_video/CMakeLists.txt delete mode 100644 user_timestamped_video/README.md delete mode 100644 user_timestamped_video/common/cli_utils.h delete mode 100644 user_timestamped_video/consumer/CMakeLists.txt delete mode 100644 user_timestamped_video/producer/CMakeLists.txt delete mode 100644 user_timestamped_video/producer/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b54342..655b7d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,5 +92,4 @@ add_subdirectory(hello_livekit/receiver hello_livekit_receiver) add_subdirectory(platform_audio) add_subdirectory(ping_pong/ping ping_pong_ping) add_subdirectory(ping_pong/pong ping_pong_pong) -add_subdirectory(user_timestamped_video) -add_subdirectory(user_data) +add_subdirectory(frame_metadata) diff --git a/README.md b/README.md index a97340a..c5e4e18 100644 --- a/README.md +++ b/README.md @@ -73,26 +73,6 @@ For example: ./build/basic_room/basic_room --url --token ``` -### PlatformAudio - -The `platform_audio` examples show microphone capture and speaker playout using -WebRTC's platform Audio Device Module: - -```bash -./build/platform_audio/player/PlatformAudioPlayer -./build/platform_audio/sender/PlatformAudioSender -``` - -### UserData - -The `user_data` example shows how to attach small sensor readings to video -frames using SDK `v1.3.0` frame metadata: - -```bash -./build/user_data/producer/UserDataProducer -./build/user_data/consumer/UserDataConsumer -``` - ### Supported platforms Prebuilt SDKs are downloaded automatically for: diff --git a/user_data/CMakeLists.txt b/frame_metadata/CMakeLists.txt similarity index 77% rename from user_data/CMakeLists.txt rename to frame_metadata/CMakeLists.txt index 8739984..5660929 100644 --- a/user_data/CMakeLists.txt +++ b/frame_metadata/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_library(user_data_support STATIC +add_library(frame_metadata_support STATIC common/json_converters.cpp common/json_converters.h common/cli_utils.h @@ -20,8 +20,8 @@ add_library(user_data_support STATIC common/messages.h ) -target_include_directories(user_data_support PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/common) -target_link_libraries(user_data_support PUBLIC nlohmann_json::nlohmann_json) +target_include_directories(frame_metadata_support PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/common) +target_link_libraries(frame_metadata_support PUBLIC nlohmann_json::nlohmann_json) add_subdirectory(producer) add_subdirectory(consumer) diff --git a/frame_metadata/README.md b/frame_metadata/README.md new file mode 100644 index 0000000..710a5d0 --- /dev/null +++ b/frame_metadata/README.md @@ -0,0 +1,50 @@ +# Frame Metadata Example + +This example is split into two executables and demonstrates LiveKit C++ SDK video frame metadata feature: + +- `frame_metadata_producer` publishes a synthetic video track named `"sensor-camera"`. +- Each video frame attaches additional data via the frame metadata features of the SDK: + + | Field | Description | + | --- | --- | + | `user_timestamp_us` | Wall-clock capture time (us) | + | `frame_id` | Monotonic frame counter | + | `user_data` | Free-form user data, in this example it is used to send a JSON temperature reading correlated with the video frame | + +- `frame_metadata_consumer` subscribes to the remote track by name and reads the + metadata through `setOnVideoFrameEventCallback`. + +Run them in the same room with different participant identities. `LIVEKIT_URL` +defaults to `ws://localhost:7880` when unset: + +```sh +export LIVEKIT_TOKEN= +./build/frame_metadata/producer/frame_metadata_producer + +export LIVEKIT_TOKEN= +./build/frame_metadata/consumer/frame_metadata_consumer +``` + +You can also pass URL and token explicitly: + +```sh +./build/frame_metadata/producer/frame_metadata_producer ws://localhost:7880 +./build/frame_metadata/consumer/frame_metadata_consumer ws://localhost:7880 +``` + +Metadata notes for this example: + +- `frame_id` and `user_ts_us` come from `VideoFrameMetadata` and are the values + to compare end to end. +- `user_data` is free-form frame metadata. This example uses it only for the + temperature reading, similar to a robotics or sensor-fusion camera feed, in the following format: + +```json +{"temperature_c":23.4} +``` + +- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`. +- `capture_ts_us` on the consumer is the received WebRTC frame timestamp. +- Producer and consumer `capture_ts_us` values are not expected to match exactly, + because WebRTC may translate frame timestamps onto its own internal + capture-time timeline before delivery. diff --git a/user_data/common/cli_utils.h b/frame_metadata/common/cli_utils.h similarity index 77% rename from user_data/common/cli_utils.h rename to frame_metadata/common/cli_utils.h index cbd0e7d..3f37d68 100644 --- a/user_data/common/cli_utils.h +++ b/frame_metadata/common/cli_utils.h @@ -24,7 +24,9 @@ #include #include -namespace user_data { +namespace frame_metadata { + +inline constexpr char kDefaultLiveKitUrl[] = "ws://localhost:7880"; enum class ParseResult { Ok, Help, Error }; @@ -53,9 +55,15 @@ inline std::string getenvOrEmpty(const char* name) { inline void printUsage(const char* program) { std::cerr << "Usage:\n" - << " " << program << " \n" - << "or:\n" - << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program << "\n"; + << " " << program << " [ ]\n" + << "\n" + << "Environment:\n" + << " LIVEKIT_URL defaults to " << kDefaultLiveKitUrl << " when unset\n" + << " LIVEKIT_TOKEN required unless passed as the second argument\n" + << "\n" + << "Example:\n" + << " export LIVEKIT_TOKEN=\n" + << " " << program << "\n"; } inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) { @@ -88,7 +96,11 @@ inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) { return ParseResult::Error; } - return (options.url.empty() || options.token.empty()) ? ParseResult::Error : ParseResult::Ok; + if (options.url.empty()) { + options.url = kDefaultLiveKitUrl; + } + + return options.token.empty() ? ParseResult::Error : ParseResult::Ok; } inline std::vector toPayload(const std::string& text) { @@ -99,4 +111,4 @@ inline std::string toString(const std::vector& payload) { return std::string(payload.begin(), payload.end()); } -} // namespace user_data +} // namespace frame_metadata diff --git a/user_data/common/constants.h b/frame_metadata/common/constants.h similarity index 82% rename from user_data/common/constants.h rename to frame_metadata/common/constants.h index 1482884..5d443a6 100644 --- a/user_data/common/constants.h +++ b/frame_metadata/common/constants.h @@ -16,12 +16,10 @@ #pragma once -namespace user_data { +namespace frame_metadata { inline constexpr char kVideoTrackName[] = "sensor-camera"; -inline constexpr char kFrameIdKey[] = "frame_id"; -inline constexpr char kTimestampUsKey[] = "timestamp_us"; inline constexpr char kTemperatureCKey[] = "temperature_c"; -} // namespace user_data +} // namespace frame_metadata diff --git a/user_data/common/json_converters.cpp b/frame_metadata/common/json_converters.cpp similarity index 82% rename from user_data/common/json_converters.cpp rename to frame_metadata/common/json_converters.cpp index f02a38a..847048e 100644 --- a/user_data/common/json_converters.cpp +++ b/frame_metadata/common/json_converters.cpp @@ -21,12 +21,10 @@ #include "constants.h" -namespace user_data { +namespace frame_metadata { std::string sensorReadingToJson(const SensorReading& reading) { nlohmann::json json; - json[kFrameIdKey] = reading.frame_id; - json[kTimestampUsKey] = reading.timestamp_us; json[kTemperatureCKey] = reading.temperature_c; return json.dump(); } @@ -36,8 +34,6 @@ SensorReading sensorReadingFromJson(const std::string& json_text) { const auto json = nlohmann::json::parse(json_text); SensorReading reading; - reading.frame_id = json.at(kFrameIdKey).get(); - reading.timestamp_us = json.at(kTimestampUsKey).get(); reading.temperature_c = json.at(kTemperatureCKey).get(); return reading; } catch (const nlohmann::json::exception& error) { @@ -45,4 +41,4 @@ SensorReading sensorReadingFromJson(const std::string& json_text) { } } -} // namespace user_data +} // namespace frame_metadata diff --git a/user_data/common/json_converters.h b/frame_metadata/common/json_converters.h similarity index 93% rename from user_data/common/json_converters.h rename to frame_metadata/common/json_converters.h index 72ff49a..eb7f175 100644 --- a/user_data/common/json_converters.h +++ b/frame_metadata/common/json_converters.h @@ -20,9 +20,9 @@ #include "messages.h" -namespace user_data { +namespace frame_metadata { std::string sensorReadingToJson(const SensorReading& reading); SensorReading sensorReadingFromJson(const std::string& json_text); -} // namespace user_data +} // namespace frame_metadata diff --git a/user_data/common/messages.h b/frame_metadata/common/messages.h similarity index 83% rename from user_data/common/messages.h rename to frame_metadata/common/messages.h index fe87602..c96c08a 100644 --- a/user_data/common/messages.h +++ b/frame_metadata/common/messages.h @@ -16,14 +16,10 @@ #pragma once -#include - -namespace user_data { +namespace frame_metadata { struct SensorReading { - std::uint32_t frame_id = 0; - std::uint64_t timestamp_us = 0; double temperature_c = 0.0; }; -} // namespace user_data +} // namespace frame_metadata diff --git a/user_data/producer/CMakeLists.txt b/frame_metadata/consumer/CMakeLists.txt similarity index 74% rename from user_data/producer/CMakeLists.txt rename to frame_metadata/consumer/CMakeLists.txt index ece31de..add7e16 100644 --- a/user_data/producer/CMakeLists.txt +++ b/frame_metadata/consumer/CMakeLists.txt @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_executable(UserDataProducer +add_executable(frame_metadata_consumer main.cpp ) -target_link_libraries(UserDataProducer PRIVATE user_data_support ${LIVEKIT_CORE_TARGET}) +target_link_libraries(frame_metadata_consumer PRIVATE frame_metadata_support ${LIVEKIT_CORE_TARGET}) -livekit_copy_windows_runtime_dlls(UserDataProducer) +livekit_copy_windows_runtime_dlls(frame_metadata_consumer) diff --git a/user_timestamped_video/consumer/main.cpp b/frame_metadata/consumer/main.cpp similarity index 54% rename from user_timestamped_video/consumer/main.cpp rename to frame_metadata/consumer/main.cpp index f74dbef..2801132 100644 --- a/user_timestamped_video/consumer/main.cpp +++ b/frame_metadata/consumer/main.cpp @@ -14,57 +14,55 @@ * limitations under the License. */ -/// UserTimestampedVideoConsumer +/// frame_metadata_consumer /// /// Receives remote video frames via `Room::setOnVideoFrameEventCallback()` and -/// logs any `VideoFrameMetadata::user_timestamp_us` and `frame_id` values that -/// arrive. Pair with `UserTimestampedVideoProducer` running in another process. +/// logs `VideoFrameMetadata::user_timestamp_us`, `frame_id`, and `user_data`. /// /// Usage: -/// UserTimestampedVideoConsumer -/// [--with-user-timestamp|--without-user-timestamp] +/// frame_metadata_consumer [ ] /// -/// Or via environment variables: -/// LIVEKIT_URL, LIVEKIT_TOKEN +/// Or via environment variables (LIVEKIT_URL defaults to ws://localhost:7880): +/// export LIVEKIT_TOKEN= +/// frame_metadata_consumer #include #include +#include #include #include #include +#include #include #include #include -#include "../common/cli_utils.h" +#include "cli_utils.h" +#include "constants.h" +#include "json_converters.h" #include "livekit/livekit.h" +#include "messages.h" using namespace livekit; namespace { -constexpr const char* kTrackName = "timestamped-camera"; - -std::string formatUserTimestamp(const std::optional& metadata) { - if (!metadata || !metadata->user_timestamp_us.has_value()) { - return "n/a"; +std::optional parseSensorReading( + const std::optional& metadata) { + if (!metadata || !metadata->user_data.has_value()) { + return std::nullopt; } - return std::to_string(*metadata->user_timestamp_us); -} - -std::string formatFrameId(const std::optional& metadata) { - if (!metadata || !metadata->frame_id.has_value()) { - return "n/a"; + try { + return frame_metadata::sensorReadingFromJson(frame_metadata::toString(*metadata->user_data)); + } catch (const std::exception&) { + return std::nullopt; } - - return std::to_string(*metadata->frame_id); } -class UserTimestampedVideoConsumerDelegate : public RoomDelegate { +class FrameMetadataConsumerDelegate : public RoomDelegate { public: - UserTimestampedVideoConsumerDelegate(Room& room, bool read_user_timestamp) - : room_(room), read_user_timestamp_(read_user_timestamp) {} + explicit FrameMetadataConsumerDelegate(Room& room) : room_(room) {} void registerExistingParticipants() { for (const auto& weak_participant : room_.remoteParticipants()) { @@ -91,7 +89,7 @@ class UserTimestampedVideoConsumerDelegate : public RoomDelegate { } const std::string identity = event.participant->identity(); - room_.clearOnVideoFrameCallback(identity, std::string(kTrackName)); + room_.clearOnVideoFrameCallback(identity, std::string(frame_metadata::kVideoTrackName)); { std::lock_guard lock(mutex_); @@ -113,33 +111,37 @@ class UserTimestampedVideoConsumerDelegate : public RoomDelegate { VideoStream::Options stream_options; stream_options.format = VideoBufferType::RGBA; - if (read_user_timestamp_) { - room_.setOnVideoFrameEventCallback( - identity, std::string(kTrackName), - [identity](const VideoFrameEvent& event) { - std::cout << "[consumer] from=" << identity << " size=" << event.frame.width() << "x" - << event.frame.height() << " capture_ts_us=" << event.timestamp_us - << " metadata_frame_id=" << formatFrameId(event.metadata) - << " user_ts_us=" << formatUserTimestamp(event.metadata) - << " rotation=" << static_cast(event.rotation) << "\n"; - }, - stream_options); - } else { - room_.setOnVideoFrameCallback( - identity, std::string(kTrackName), - [identity](const VideoFrame& frame, const std::int64_t timestamp_us) { - std::cout << "[consumer] from=" << identity << " size=" << frame.width() << "x" << frame.height() - << " capture_ts_us=" << timestamp_us << " metadata_frame_id=ignored user_ts_us=ignored\n"; - }, - stream_options); - } - - std::cout << "[consumer] listening for video frames from " << identity << " track=\"" << kTrackName - << "\" with user timestamp " << (read_user_timestamp_ ? "enabled" : "ignored") << "\n"; + room_.setOnVideoFrameEventCallback( + identity, std::string(frame_metadata::kVideoTrackName), + [](const VideoFrameEvent& event) { + if (!event.metadata || !event.metadata->frame_id || !event.metadata->user_timestamp_us || + !event.metadata->user_data) { + return; + } + + const std::uint32_t frame_id = *event.metadata->frame_id; + if (frame_id % 5 != 0) { + return; + } + + const auto reading = parseSensorReading(event.metadata); + if (!reading) { + return; + } + + std::cout << std::fixed << std::setprecision(2) << "[consumer] frame_id=" << frame_id + << " capture_ts_us=" << event.timestamp_us + << " user_ts_us=" << *event.metadata->user_timestamp_us + << " temperature_c=" << reading->temperature_c + << " user_data_bytes=" << event.metadata->user_data->size() << "\n"; + }, + stream_options); + + std::cout << "[consumer] listening for video frames from " << identity << " track=\"" + << frame_metadata::kVideoTrackName << "\" with frame metadata\n"; } Room& room_; - bool read_user_timestamp_; std::mutex mutex_; std::unordered_set registered_identities_; }; @@ -147,15 +149,15 @@ class UserTimestampedVideoConsumerDelegate : public RoomDelegate { } // namespace int main(int argc, char* argv[]) { - user_timestamped_video::CliOptions cli_options; + frame_metadata::CliOptions cli_options; - const user_timestamped_video::ParseResult parse_result = user_timestamped_video::parseArgs(argc, argv, cli_options); - if (parse_result != user_timestamped_video::ParseResult::Ok) { - user_timestamped_video::printUsage(argv[0]); - return parse_result == user_timestamped_video::ParseResult::Help ? 0 : 1; + const frame_metadata::ParseResult parse_result = frame_metadata::parseArgs(argc, argv, cli_options); + if (parse_result != frame_metadata::ParseResult::Ok) { + frame_metadata::printUsage(argv[0]); + return parse_result == frame_metadata::ParseResult::Help ? 0 : 1; } - user_timestamped_video::installSignalHandlers(); + frame_metadata::installSignalHandlers(); livekit::initialize(livekit::LogLevel::Info); int exit_code = 0; @@ -166,7 +168,7 @@ int main(int argc, char* argv[]) { options.auto_subscribe = true; options.dynacast = false; - UserTimestampedVideoConsumerDelegate delegate(room, cli_options.use_user_timestamp); + FrameMetadataConsumerDelegate delegate(room); room.setDelegate(&delegate); std::cout << "[consumer] connecting to " << cli_options.url << "\n"; @@ -176,21 +178,20 @@ int main(int argc, char* argv[]) { } else { if (auto lp = room.localParticipant().lock()) { std::cout << "[consumer] connected as " << (lp ? lp->identity() : std::string("")) << " to room '" - << room.roomInfo().name << "' with user timestamp " - << (cli_options.use_user_timestamp ? "enabled" : "ignored") << "\n"; + << room.roomInfo().name << "'\n"; } else { throw std::runtime_error("unable to lock local participant"); } delegate.registerExistingParticipants(); - while (user_timestamped_video::isRunning()) { + while (frame_metadata::isRunning()) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } for (const auto& weak_participant : room.remoteParticipants()) { if (auto participant = weak_participant.lock()) { - room.clearOnVideoFrameCallback(participant->identity(), std::string(kTrackName)); + room.clearOnVideoFrameCallback(participant->identity(), std::string(frame_metadata::kVideoTrackName)); } else { throw std::runtime_error("unable to lock provided remote participant"); } diff --git a/user_data/consumer/CMakeLists.txt b/frame_metadata/producer/CMakeLists.txt similarity index 74% rename from user_data/consumer/CMakeLists.txt rename to frame_metadata/producer/CMakeLists.txt index f2b0250..934af55 100644 --- a/user_data/consumer/CMakeLists.txt +++ b/frame_metadata/producer/CMakeLists.txt @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_executable(UserDataConsumer +add_executable(frame_metadata_producer main.cpp ) -target_link_libraries(UserDataConsumer PRIVATE user_data_support ${LIVEKIT_CORE_TARGET}) +target_link_libraries(frame_metadata_producer PRIVATE frame_metadata_support ${LIVEKIT_CORE_TARGET}) -livekit_copy_windows_runtime_dlls(UserDataConsumer) +livekit_copy_windows_runtime_dlls(frame_metadata_producer) diff --git a/user_data/producer/main.cpp b/frame_metadata/producer/main.cpp similarity index 61% rename from user_data/producer/main.cpp rename to frame_metadata/producer/main.cpp index 944d319..1232547 100644 --- a/user_data/producer/main.cpp +++ b/frame_metadata/producer/main.cpp @@ -14,17 +14,17 @@ * limitations under the License. */ -/// UserDataProducer +/// frame_metadata_producer /// -/// Publishes a synthetic camera track. Each video frame carries v1.3.0 frame -/// metadata: `frame_id`, `user_timestamp_us`, and a compact temperature reading -/// in `user_data`. +/// Publishes a synthetic camera track and stamps each frame with v1.3.0 frame +/// metadata: `user_timestamp_us`, `frame_id`, and compact `user_data`. /// /// Usage: -/// UserDataProducer +/// frame_metadata_producer [ ] /// -/// Or via environment variables: -/// LIVEKIT_URL, LIVEKIT_TOKEN +/// Or via environment variables (LIVEKIT_URL defaults to ws://localhost:7880): +/// export LIVEKIT_TOKEN= +/// frame_metadata_producer #include #include @@ -79,15 +79,15 @@ void fillFrame(VideoFrame& frame, double temperature_c) { } // namespace int main(int argc, char* argv[]) { - user_data::CliOptions cli_options; + frame_metadata::CliOptions cli_options; - const user_data::ParseResult parse_result = user_data::parseArgs(argc, argv, cli_options); - if (parse_result != user_data::ParseResult::Ok) { - user_data::printUsage(argv[0]); - return parse_result == user_data::ParseResult::Help ? 0 : 1; + const frame_metadata::ParseResult parse_result = frame_metadata::parseArgs(argc, argv, cli_options); + if (parse_result != frame_metadata::ParseResult::Ok) { + frame_metadata::printUsage(argv[0]); + return parse_result == frame_metadata::ParseResult::Help ? 0 : 1; } - user_data::installSignalHandlers(); + frame_metadata::installSignalHandlers(); livekit::initialize(livekit::LogLevel::Info); int exit_code = 0; @@ -103,28 +103,29 @@ int main(int argc, char* argv[]) { std::cerr << "[producer] failed to connect\n"; exit_code = 1; } else { - std::shared_ptr video_track; + { + auto lp = room.localParticipant().lock(); + std::cout << "[producer] connected as " << (lp ? lp->identity() : std::string("")) << " to room '" + << room.roomInfo().name << "'\n"; + } + auto source = std::make_shared(kFrameWidth, kFrameHeight); + auto track = LocalVideoTrack::createLocalVideoTrack(frame_metadata::kVideoTrackName, source); try { + TrackPublishOptions publish_options; + publish_options.source = TrackSource::SOURCE_CAMERA; + publish_options.frame_metadata_features = FrameMetadataFeatures{}; + publish_options.frame_metadata_features->user_timestamp = true; + publish_options.frame_metadata_features->frame_id = true; + publish_options.frame_metadata_features->user_data = true; + { auto lp = room.localParticipant().lock(); if (!lp) throw std::runtime_error("local participant unavailable"); - - std::cout << "[producer] connected as " << lp->identity() << " to room '" << room.roomInfo().name << "'\n"; - - video_track = LocalVideoTrack::createLocalVideoTrack(user_data::kVideoTrackName, source); - - TrackPublishOptions publish_options; - publish_options.source = TrackSource::SOURCE_CAMERA; - publish_options.frame_metadata_features = FrameMetadataFeatures{}; - publish_options.frame_metadata_features->user_timestamp = true; - publish_options.frame_metadata_features->frame_id = true; - publish_options.frame_metadata_features->user_data = true; - - lp->publishTrack(video_track, publish_options); - std::cout << "[producer] published video track \"" << user_data::kVideoTrackName - << "\" with timestamp, frame id, and user data metadata\n"; + lp->publishTrack(track, publish_options); + std::cout << "[producer] published camera track \"" << frame_metadata::kVideoTrackName + << "\" with frame metadata\n"; } VideoFrame frame = VideoFrame::create(kFrameWidth, kFrameHeight, VideoBufferType::BGRA); @@ -132,30 +133,30 @@ int main(int argc, char* argv[]) { std::uint32_t frame_id = 0; auto next_frame_at = std::chrono::steady_clock::now(); - std::cout << "[producer] sending synthetic video with inline temperature metadata; Ctrl-C to exit\n"; - while (user_data::isRunning()) { - user_data::SensorReading reading; - reading.frame_id = frame_id; - reading.timestamp_us = nowEpochUs(); - reading.temperature_c = simulatedTemperatureC(frame_id); + while (frame_metadata::isRunning()) { + const std::uint64_t user_timestamp_us = nowEpochUs(); + const double temperature_c = simulatedTemperatureC(frame_id); - fillFrame(frame, reading.temperature_c); + fillFrame(frame, temperature_c); VideoCaptureOptions capture_options; capture_options.timestamp_us = static_cast( std::chrono::duration_cast(std::chrono::steady_clock::now() - capture_start) .count()); capture_options.rotation = VideoRotation::VIDEO_ROTATION_0; + capture_options.metadata = VideoFrameMetadata{}; - capture_options.metadata->user_timestamp_us = reading.timestamp_us; - capture_options.metadata->frame_id = reading.frame_id; - capture_options.metadata->user_data = user_data::toPayload(user_data::sensorReadingToJson(reading)); + capture_options.metadata->user_timestamp_us = user_timestamp_us; + capture_options.metadata->frame_id = frame_id; + capture_options.metadata->user_data = frame_metadata::toPayload( + frame_metadata::sensorReadingToJson(frame_metadata::SensorReading{temperature_c})); source->captureFrame(frame, capture_options); if (frame_id % 5 == 0) { - std::cout << std::fixed << std::setprecision(2) << "[producer] frame_id=" << reading.frame_id - << " user_ts_us=" << reading.timestamp_us << " temperature_c=" << reading.temperature_c + std::cout << std::fixed << std::setprecision(2) << "[producer] frame_id=" << frame_id + << " capture_ts_us=" << capture_options.timestamp_us + << " user_ts_us=" << user_timestamp_us << " temperature_c=" << temperature_c << " user_data_bytes=" << capture_options.metadata->user_data->size() << "\n"; } @@ -169,8 +170,8 @@ int main(int argc, char* argv[]) { } if (auto lp = room.localParticipant().lock()) { - if (video_track && video_track->publication()) { - lp->unpublishTrack(video_track->publication()->sid()); + if (lp && track->publication()) { + lp->unpublishTrack(track->publication()->sid()); } } } diff --git a/user_data/README.md b/user_data/README.md deleted file mode 100644 index 8804b91..0000000 --- a/user_data/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# UserData - -This example shows how to pair small application data with video frames: - -- `UserDataProducer` publishes a synthetic video track named `"sensor-camera"`. - Each frame carries `VideoCaptureOptions::metadata.frame_id` and - `VideoCaptureOptions::metadata.user_timestamp_us`. -- The producer also attaches a small JSON temperature reading to - `VideoCaptureOptions::metadata.user_data`. -- `UserDataConsumer` subscribes to the video track and parses the inline - `user_data` payload on each received frame. - -Run them in the same room with different participant identities: - -```sh -LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserDataProducer -LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserDataConsumer -``` - -Requirements: - -- LiveKit C++ SDK `v1.3.0` or newer. This example uses packet-trailer frame IDs - and user data in `VideoFrameMetadata`. -- To pin the SDK version when configuring the examples, pass - `-DLIVEKIT_SDK_VERSION=1.3.0` to CMake. - -The data payload is intentionally small: - -```json -{"frame_id":42,"timestamp_us":1782930000000000,"temperature_c":23.4} -``` - -In a robotics or sensor-fusion application, this lets a small telemetry value -travel with the frame it describes without requiring a companion data track. diff --git a/user_data/consumer/main.cpp b/user_data/consumer/main.cpp deleted file mode 100644 index c9eeef5..0000000 --- a/user_data/consumer/main.cpp +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright 2026 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/// UserDataConsumer -/// -/// Receives video frame events with v1.3.0 metadata and parses small -/// temperature readings from `VideoFrameMetadata::user_data`. -/// -/// Usage: -/// UserDataConsumer -/// -/// Or via environment variables: -/// LIVEKIT_URL, LIVEKIT_TOKEN - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cli_utils.h" -#include "constants.h" -#include "json_converters.h" -#include "livekit/livekit.h" -#include "messages.h" - -using namespace livekit; - -namespace { - -std::optional metadataFrameId(const std::optional& metadata) { - if (!metadata || !metadata->frame_id.has_value()) { - return std::nullopt; - } - - return *metadata->frame_id; -} - -std::optional metadataSensorReading(const std::optional& metadata) { - if (!metadata || !metadata->user_data.has_value()) { - return std::nullopt; - } - - return user_data::sensorReadingFromJson(user_data::toString(*metadata->user_data)); -} - -std::string formatOptionalTimestamp(std::optional timestamp_us) { - return timestamp_us ? std::to_string(*timestamp_us) : std::string("n/a"); -} - -class UserDataConsumerDelegate : public RoomDelegate { -public: - explicit UserDataConsumerDelegate(Room& room) : room_(room) {} - - void registerExistingParticipants() { - for (const auto& weak_participant : room_.remoteParticipants()) { - if (auto participant = weak_participant.lock()) { - registerRemoteVideoCallback(participant->identity()); - } else { - throw std::runtime_error("unable to lock provided remote participant"); - } - } - } - - void clearAllCallbacks() { - std::unordered_set identities; - { - std::lock_guard lock(mutex_); - identities.swap(registered_identities_); - } - - for (const auto& identity : identities) { - room_.clearOnVideoFrameCallback(identity, std::string(user_data::kVideoTrackName)); - } - } - - void onParticipantConnected(Room&, const ParticipantConnectedEvent& event) override { - if (!event.participant) { - return; - } - - std::cout << "[consumer] participant connected: " << event.participant->identity() << "\n"; - registerRemoteVideoCallback(event.participant->identity()); - } - - void onParticipantDisconnected(Room&, const ParticipantDisconnectedEvent& event) override { - if (!event.participant) { - return; - } - - const std::string identity = event.participant->identity(); - room_.clearOnVideoFrameCallback(identity, std::string(user_data::kVideoTrackName)); - { - std::lock_guard lock(mutex_); - registered_identities_.erase(identity); - } - - std::cout << "[consumer] participant disconnected: " << identity << "\n"; - } - -private: - void registerRemoteVideoCallback(const std::string& identity) { - { - std::lock_guard lock(mutex_); - if (!registered_identities_.insert(identity).second) { - return; - } - } - - VideoStream::Options stream_options; - stream_options.format = VideoBufferType::RGBA; - - room_.setOnVideoFrameEventCallback( - identity, std::string(user_data::kVideoTrackName), - [identity](const VideoFrameEvent& event) { - try { - const auto frame_id = metadataFrameId(event.metadata); - const auto reading = metadataSensorReading(event.metadata); - - std::cout << "[consumer] from=" << identity << " size=" << event.frame.width() << "x" - << event.frame.height() << " capture_ts_us=" << event.timestamp_us - << " metadata_frame_id=" << (frame_id ? std::to_string(*frame_id) : std::string("n/a")) - << " user_ts_us=" << formatOptionalTimestamp(event.metadata ? event.metadata->user_timestamp_us - : std::nullopt); - - if (reading) { - std::cout << std::fixed << std::setprecision(2) << " reading_frame_id=" << reading->frame_id - << " temperature_c=" << reading->temperature_c - << " reading_ts_us=" << reading->timestamp_us - << " user_data_bytes=" << event.metadata->user_data->size(); - } else { - std::cout << " user_data=n/a"; - } - - std::cout << "\n"; - } catch (const std::exception& error) { - std::cerr << "[consumer] failed to process frame metadata from " << identity << ": " << error.what() - << "\n"; - } - }, - stream_options); - - std::cout << "[consumer] listening for video frames from " << identity << " track=\"" - << user_data::kVideoTrackName << "\"\n"; - } - - Room& room_; - std::mutex mutex_; - std::unordered_set registered_identities_; -}; - -} // namespace - -int main(int argc, char* argv[]) { - user_data::CliOptions cli_options; - - const user_data::ParseResult parse_result = user_data::parseArgs(argc, argv, cli_options); - if (parse_result != user_data::ParseResult::Ok) { - user_data::printUsage(argv[0]); - return parse_result == user_data::ParseResult::Help ? 0 : 1; - } - - user_data::installSignalHandlers(); - - livekit::initialize(livekit::LogLevel::Info); - int exit_code = 0; - - { - Room room; - RoomOptions options; - options.auto_subscribe = true; - options.dynacast = false; - - UserDataConsumerDelegate delegate(room); - room.setDelegate(&delegate); - - std::cout << "[consumer] connecting to " << cli_options.url << "\n"; - if (!room.connect(cli_options.url, cli_options.token, options)) { - std::cerr << "[consumer] failed to connect\n"; - exit_code = 1; - } else { - if (auto lp = room.localParticipant().lock()) { - std::cout << "[consumer] connected as " << lp->identity() << " to room '" << room.roomInfo().name << "'\n"; - } else { - throw std::runtime_error("unable to lock local participant"); - } - - delegate.registerExistingParticipants(); - - while (user_data::isRunning()) { - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } - - delegate.clearAllCallbacks(); - } - - room.setDelegate(nullptr); - } - - livekit::shutdown(); - return exit_code; -} diff --git a/user_timestamped_video/CMakeLists.txt b/user_timestamped_video/CMakeLists.txt deleted file mode 100644 index 6fdcdd8..0000000 --- a/user_timestamped_video/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2026 LiveKit, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_subdirectory(producer) -add_subdirectory(consumer) diff --git a/user_timestamped_video/README.md b/user_timestamped_video/README.md deleted file mode 100644 index 3846aca..0000000 --- a/user_timestamped_video/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# UserTimestampedVideo - -This example is split into two executables and can demonstrate all four -producer/consumer combinations: - -- `UserTimestampedVideoProducer` publishes a synthetic video track named - `"timestamped-camera"` and stamps each frame with both - `VideoCaptureOptions::metadata.user_timestamp_us` and - `VideoCaptureOptions::metadata.frame_id`. -- `UserTimestampedVideoConsumer` subscribes to the remote - `"timestamped-camera"` track by name with either the rich or legacy callback - path. - -Run them in the same room with different participant identities: - -```sh -LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoProducer -LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoConsumer -``` - -Requirements: - -- LiveKit C++ SDK `v1.3.0` or newer. This example uses `VideoFrameMetadata` - frame IDs and `setOnVideoFrameEventCallback`, which are not available in - older SDK releases. -- To pin the SDK version when configuring the examples, pass - `-DLIVEKIT_SDK_VERSION=1.3.0` to CMake. - -Flags: - -- Producer default: sends user timestamps and frame IDs -- Producer `--with-user-timestamp`: explicitly sends user timestamps and frame IDs -- Producer `--without-user-timestamp`: does not send frame metadata -- Consumer default: reads metadata through `setOnVideoFrameEventCallback` -- Consumer `--with-user-timestamp`: explicitly reads metadata through - `setOnVideoFrameEventCallback` -- Consumer `--without-user-timestamp`: ignores metadata through the legacy - `setOnVideoFrameCallback` - -Matrix: - -```sh -# 1. Producer sends, consumer reads -./UserTimestampedVideoProducer -./UserTimestampedVideoConsumer - -# 2. Producer sends, consumer ignores -./UserTimestampedVideoProducer -./UserTimestampedVideoConsumer --without-user-timestamp - -# 3. Producer does not send, consumer ignores -./UserTimestampedVideoProducer --without-user-timestamp -./UserTimestampedVideoConsumer --without-user-timestamp - -# 4. Producer does not send, consumer reads -./UserTimestampedVideoProducer --without-user-timestamp -./UserTimestampedVideoConsumer -``` - -Timestamp note: - -- `user_ts_us` is application metadata and is the value to compare end to end. -- `metadata_frame_id` is application metadata and should match the producer's - frame counter when the frame metadata path is enabled. -- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`. -- `capture_ts_us` on the consumer is the received WebRTC frame timestamp. -- Producer and consumer `capture_ts_us` values are not expected to match exactly, - because WebRTC may translate frame timestamps onto its own internal - capture-time timeline before delivery. diff --git a/user_timestamped_video/common/cli_utils.h b/user_timestamped_video/common/cli_utils.h deleted file mode 100644 index f6102ad..0000000 --- a/user_timestamped_video/common/cli_utils.h +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2026 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace user_timestamped_video { - -enum class ParseResult { Ok, Help, Error }; - -struct CliOptions { - std::string url; - std::string token; - bool use_user_timestamp = true; -}; - -inline std::atomic g_running{true}; - -inline void handleSignal(int) { g_running.store(false); } - -inline bool isRunning() { return g_running.load(std::memory_order_relaxed); } - -inline void installSignalHandlers() { - std::signal(SIGINT, handleSignal); -#ifdef SIGTERM - std::signal(SIGTERM, handleSignal); -#endif -} - -inline std::string getenvOrEmpty(const char* name) { - const char* value = std::getenv(name); - return value ? std::string(value) : std::string{}; -} - -inline void printUsage(const char* program) { - std::cerr << "Usage:\n" - << " " << program << " " - << "[--with-user-timestamp|--without-user-timestamp]\n" - << "or:\n" - << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program - << " [--with-user-timestamp|--without-user-timestamp]\n"; -} - -inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) { - std::vector positional; - options = CliOptions{}; - - for (int i = 1; i < argc; ++i) { - const std::string arg = argv[i]; - if (arg == "-h" || arg == "--help") { - return ParseResult::Help; - } - if (arg == "--without-user-timestamp") { - options.use_user_timestamp = false; - continue; - } - if (arg == "--with-user-timestamp") { - options.use_user_timestamp = true; - continue; - } - if (!arg.empty() && arg[0] == '-') { - return ParseResult::Error; - } - - positional.push_back(arg); - } - - if (positional.size() > 2) { - return ParseResult::Error; - } - - options.url = getenvOrEmpty("LIVEKIT_URL"); - options.token = getenvOrEmpty("LIVEKIT_TOKEN"); - - if (positional.size() == 2) { - options.url = positional[0]; - options.token = positional[1]; - } else if (positional.size() == 1) { - return ParseResult::Error; - } - - return (options.url.empty() || options.token.empty()) ? ParseResult::Error : ParseResult::Ok; -} - -} // namespace user_timestamped_video diff --git a/user_timestamped_video/consumer/CMakeLists.txt b/user_timestamped_video/consumer/CMakeLists.txt deleted file mode 100644 index d2f80a2..0000000 --- a/user_timestamped_video/consumer/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2026 LiveKit, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_executable(UserTimestampedVideoConsumer - main.cpp -) - -target_include_directories(UserTimestampedVideoConsumer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(UserTimestampedVideoConsumer PRIVATE ${LIVEKIT_CORE_TARGET}) - -livekit_copy_windows_runtime_dlls(UserTimestampedVideoConsumer) diff --git a/user_timestamped_video/producer/CMakeLists.txt b/user_timestamped_video/producer/CMakeLists.txt deleted file mode 100644 index 5c573ea..0000000 --- a/user_timestamped_video/producer/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2026 LiveKit, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_executable(UserTimestampedVideoProducer - main.cpp -) - -target_include_directories(UserTimestampedVideoProducer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(UserTimestampedVideoProducer PRIVATE ${LIVEKIT_CORE_TARGET}) - -livekit_copy_windows_runtime_dlls(UserTimestampedVideoProducer) diff --git a/user_timestamped_video/producer/main.cpp b/user_timestamped_video/producer/main.cpp deleted file mode 100644 index 2c297e4..0000000 --- a/user_timestamped_video/producer/main.cpp +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright 2026 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/// UserTimestampedVideoProducer -/// -/// Publishes a synthetic camera track and stamps each frame with -/// `VideoCaptureOptions::metadata.user_timestamp_us` and `frame_id`. Pair with -/// `UserTimestampedVideoConsumer` in another process to observe the user -/// metadata flowing end to end. -/// -/// Usage: -/// UserTimestampedVideoProducer -/// [--with-user-timestamp|--without-user-timestamp] -/// -/// Or via environment variables: -/// LIVEKIT_URL, LIVEKIT_TOKEN - -#include -#include -#include -#include -#include -#include - -#include "../common/cli_utils.h" -#include "livekit/livekit.h" - -using namespace livekit; - -namespace { - -constexpr int kFrameWidth = 640; -constexpr int kFrameHeight = 360; -constexpr int kFrameIntervalMs = 200; - -std::uint64_t nowEpochUs() { - return static_cast( - std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) - .count()); -} - -void fillFrame(VideoFrame& frame, std::uint32_t frame_index) { - const std::uint8_t blue = static_cast((frame_index * 7) % 255); - const std::uint8_t green = static_cast((frame_index * 13) % 255); - const std::uint8_t red = static_cast((frame_index * 29) % 255); - - std::uint8_t* data = frame.data(); - for (std::size_t i = 0; i < frame.dataSize(); i += 4) { - data[i + 0] = blue; - data[i + 1] = green; - data[i + 2] = red; - data[i + 3] = 255; - } -} - -} // namespace - -int main(int argc, char* argv[]) { - user_timestamped_video::CliOptions cli_options; - - const user_timestamped_video::ParseResult parse_result = user_timestamped_video::parseArgs(argc, argv, cli_options); - if (parse_result != user_timestamped_video::ParseResult::Ok) { - user_timestamped_video::printUsage(argv[0]); - return parse_result == user_timestamped_video::ParseResult::Help ? 0 : 1; - } - - user_timestamped_video::installSignalHandlers(); - - livekit::initialize(livekit::LogLevel::Info); - int exit_code = 0; - - { - Room room; - RoomOptions options; - options.auto_subscribe = true; - options.dynacast = false; - - std::cout << "[producer] connecting to " << cli_options.url << "\n"; - if (!room.connect(cli_options.url, cli_options.token, options)) { - std::cerr << "[producer] failed to connect\n"; - exit_code = 1; - } else { - { - auto lp = room.localParticipant().lock(); - std::cout << "[producer] connected as " << (lp ? lp->identity() : std::string("")) << " to room '" - << room.roomInfo().name << "'\n"; - } - - auto source = std::make_shared(kFrameWidth, kFrameHeight); - auto track = LocalVideoTrack::createLocalVideoTrack("timestamped-camera", source); - - try { - TrackPublishOptions publish_options; - publish_options.source = TrackSource::SOURCE_CAMERA; - publish_options.frame_metadata_features = FrameMetadataFeatures{}; - publish_options.frame_metadata_features->user_timestamp = cli_options.use_user_timestamp; - publish_options.frame_metadata_features->frame_id = cli_options.use_user_timestamp; - - { - auto lp = room.localParticipant().lock(); - if (!lp) throw std::runtime_error("local participant unavailable"); - lp->publishTrack(track, publish_options); - std::cout << "[producer] published camera track with frame metadata " - << (cli_options.use_user_timestamp ? "enabled" : "disabled") << "\n"; - } - - VideoFrame frame = VideoFrame::create(kFrameWidth, kFrameHeight, VideoBufferType::BGRA); - const auto capture_start = std::chrono::steady_clock::now(); - std::uint32_t frame_index = 0; - auto next_frame_at = std::chrono::steady_clock::now(); - - while (user_timestamped_video::isRunning()) { - fillFrame(frame, frame_index); - - VideoCaptureOptions capture_options; - - // a steady_clock to align with other data/video frames - capture_options.timestamp_us = static_cast( - std::chrono::duration_cast(std::chrono::steady_clock::now() - capture_start) - .count()); - capture_options.rotation = VideoRotation::VIDEO_ROTATION_0; - if (cli_options.use_user_timestamp) { - capture_options.metadata = VideoFrameMetadata{}; - capture_options.metadata->user_timestamp_us = nowEpochUs(); - capture_options.metadata->frame_id = frame_index; - } - - source->captureFrame(frame, capture_options); - - if (frame_index % 5 == 0) { - std::cout << "[producer] frame=" << frame_index << " capture_ts_us=" << capture_options.timestamp_us - << " metadata_frame_id=" - << (cli_options.use_user_timestamp ? std::to_string(*capture_options.metadata->frame_id) - : std::string("disabled")) - << " user_ts_us=" - << (cli_options.use_user_timestamp ? std::to_string(*capture_options.metadata->user_timestamp_us) - : std::string("disabled")) - << "\n"; - } - - ++frame_index; - next_frame_at += std::chrono::milliseconds(kFrameIntervalMs); - std::this_thread::sleep_until(next_frame_at); - } - } catch (const std::exception& error) { - std::cerr << "[producer] error: " << error.what() << "\n"; - exit_code = 1; - } - - // best effort unpublish - if (auto lp = room.localParticipant().lock()) { - if (lp && track->publication()) { - lp->unpublishTrack(track->publication()->sid()); - } - } - } - } - - livekit::shutdown(); - return exit_code; -}