diff --git a/CMakeLists.txt b/CMakeLists.txt index 83b916a..655b7d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,4 +92,4 @@ add_subdirectory(hello_livekit/receiver hello_livekit_receiver) add_subdirectory(platform_audio) add_subdirectory(ping_pong/ping ping_pong_ping) add_subdirectory(ping_pong/pong ping_pong_pong) -add_subdirectory(user_timestamped_video) +add_subdirectory(frame_metadata) diff --git a/README.md b/README.md index 88107bd..c5e4e18 100644 --- a/README.md +++ b/README.md @@ -73,16 +73,6 @@ For example: ./build/basic_room/basic_room --url --token ``` -### PlatformAudio - -The `platform_audio` examples show microphone capture and speaker playout using -WebRTC's platform Audio Device Module: - -```bash -./build/platform_audio/player/PlatformAudioPlayer -./build/platform_audio/sender/PlatformAudioSender -``` - ### Supported platforms Prebuilt SDKs are downloaded automatically for: diff --git a/user_timestamped_video/consumer/CMakeLists.txt b/frame_metadata/CMakeLists.txt similarity index 59% rename from user_timestamped_video/consumer/CMakeLists.txt rename to frame_metadata/CMakeLists.txt index d2f80a2..5660929 100644 --- a/user_timestamped_video/consumer/CMakeLists.txt +++ b/frame_metadata/CMakeLists.txt @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_executable(UserTimestampedVideoConsumer - main.cpp +add_library(frame_metadata_support STATIC + common/json_converters.cpp + common/json_converters.h + common/cli_utils.h + common/constants.h + common/messages.h ) -target_include_directories(UserTimestampedVideoConsumer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(UserTimestampedVideoConsumer PRIVATE ${LIVEKIT_CORE_TARGET}) +target_include_directories(frame_metadata_support PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/common) +target_link_libraries(frame_metadata_support PUBLIC nlohmann_json::nlohmann_json) -livekit_copy_windows_runtime_dlls(UserTimestampedVideoConsumer) +add_subdirectory(producer) +add_subdirectory(consumer) diff --git a/frame_metadata/README.md b/frame_metadata/README.md new file mode 100644 index 0000000..710a5d0 --- /dev/null +++ b/frame_metadata/README.md @@ -0,0 +1,50 @@ +# Frame Metadata Example + +This example is split into two executables and demonstrates LiveKit C++ SDK video frame metadata feature: + +- `frame_metadata_producer` publishes a synthetic video track named `"sensor-camera"`. +- Each video frame attaches additional data via the frame metadata features of the SDK: + + | Field | Description | + | --- | --- | + | `user_timestamp_us` | Wall-clock capture time (us) | + | `frame_id` | Monotonic frame counter | + | `user_data` | Free-form user data, in this example it is used to send a JSON temperature reading correlated with the video frame | + +- `frame_metadata_consumer` subscribes to the remote track by name and reads the + metadata through `setOnVideoFrameEventCallback`. + +Run them in the same room with different participant identities. `LIVEKIT_URL` +defaults to `ws://localhost:7880` when unset: + +```sh +export LIVEKIT_TOKEN= +./build/frame_metadata/producer/frame_metadata_producer + +export LIVEKIT_TOKEN= +./build/frame_metadata/consumer/frame_metadata_consumer +``` + +You can also pass URL and token explicitly: + +```sh +./build/frame_metadata/producer/frame_metadata_producer ws://localhost:7880 +./build/frame_metadata/consumer/frame_metadata_consumer ws://localhost:7880 +``` + +Metadata notes for this example: + +- `frame_id` and `user_ts_us` come from `VideoFrameMetadata` and are the values + to compare end to end. +- `user_data` is free-form frame metadata. This example uses it only for the + temperature reading, similar to a robotics or sensor-fusion camera feed, in the following format: + +```json +{"temperature_c":23.4} +``` + +- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`. +- `capture_ts_us` on the consumer is the received WebRTC frame timestamp. +- Producer and consumer `capture_ts_us` values are not expected to match exactly, + because WebRTC may translate frame timestamps onto its own internal + capture-time timeline before delivery. diff --git a/user_timestamped_video/common/cli_utils.h b/frame_metadata/common/cli_utils.h similarity index 68% rename from user_timestamped_video/common/cli_utils.h rename to frame_metadata/common/cli_utils.h index f6102ad..3f37d68 100644 --- a/user_timestamped_video/common/cli_utils.h +++ b/frame_metadata/common/cli_utils.h @@ -18,19 +18,21 @@ #include #include +#include #include #include #include #include -namespace user_timestamped_video { +namespace frame_metadata { + +inline constexpr char kDefaultLiveKitUrl[] = "ws://localhost:7880"; enum class ParseResult { Ok, Help, Error }; struct CliOptions { std::string url; std::string token; - bool use_user_timestamp = true; }; inline std::atomic g_running{true}; @@ -53,11 +55,15 @@ inline std::string getenvOrEmpty(const char* name) { inline void printUsage(const char* program) { std::cerr << "Usage:\n" - << " " << program << " " - << "[--with-user-timestamp|--without-user-timestamp]\n" - << "or:\n" - << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program - << " [--with-user-timestamp|--without-user-timestamp]\n"; + << " " << program << " [ ]\n" + << "\n" + << "Environment:\n" + << " LIVEKIT_URL defaults to " << kDefaultLiveKitUrl << " when unset\n" + << " LIVEKIT_TOKEN required unless passed as the second argument\n" + << "\n" + << "Example:\n" + << " export LIVEKIT_TOKEN=\n" + << " " << program << "\n"; } inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) { @@ -69,14 +75,6 @@ inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) { if (arg == "-h" || arg == "--help") { return ParseResult::Help; } - if (arg == "--without-user-timestamp") { - options.use_user_timestamp = false; - continue; - } - if (arg == "--with-user-timestamp") { - options.use_user_timestamp = true; - continue; - } if (!arg.empty() && arg[0] == '-') { return ParseResult::Error; } @@ -98,7 +96,19 @@ inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) { return ParseResult::Error; } - return (options.url.empty() || options.token.empty()) ? ParseResult::Error : ParseResult::Ok; + if (options.url.empty()) { + options.url = kDefaultLiveKitUrl; + } + + return options.token.empty() ? ParseResult::Error : ParseResult::Ok; +} + +inline std::vector toPayload(const std::string& text) { + return std::vector(text.begin(), text.end()); +} + +inline std::string toString(const std::vector& payload) { + return std::string(payload.begin(), payload.end()); } -} // namespace user_timestamped_video +} // namespace frame_metadata diff --git a/frame_metadata/common/constants.h b/frame_metadata/common/constants.h new file mode 100644 index 0000000..5d443a6 --- /dev/null +++ b/frame_metadata/common/constants.h @@ -0,0 +1,25 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace frame_metadata { + +inline constexpr char kVideoTrackName[] = "sensor-camera"; + +inline constexpr char kTemperatureCKey[] = "temperature_c"; + +} // namespace frame_metadata diff --git a/frame_metadata/common/json_converters.cpp b/frame_metadata/common/json_converters.cpp new file mode 100644 index 0000000..847048e --- /dev/null +++ b/frame_metadata/common/json_converters.cpp @@ -0,0 +1,44 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "json_converters.h" + +#include +#include + +#include "constants.h" + +namespace frame_metadata { + +std::string sensorReadingToJson(const SensorReading& reading) { + nlohmann::json json; + json[kTemperatureCKey] = reading.temperature_c; + return json.dump(); +} + +SensorReading sensorReadingFromJson(const std::string& json_text) { + try { + const auto json = nlohmann::json::parse(json_text); + + SensorReading reading; + reading.temperature_c = json.at(kTemperatureCKey).get(); + return reading; + } catch (const nlohmann::json::exception& error) { + throw std::runtime_error(std::string("Failed to parse sensor reading JSON: ") + error.what()); + } +} + +} // namespace frame_metadata diff --git a/frame_metadata/common/json_converters.h b/frame_metadata/common/json_converters.h new file mode 100644 index 0000000..eb7f175 --- /dev/null +++ b/frame_metadata/common/json_converters.h @@ -0,0 +1,28 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "messages.h" + +namespace frame_metadata { + +std::string sensorReadingToJson(const SensorReading& reading); +SensorReading sensorReadingFromJson(const std::string& json_text); + +} // namespace frame_metadata diff --git a/frame_metadata/common/messages.h b/frame_metadata/common/messages.h new file mode 100644 index 0000000..c96c08a --- /dev/null +++ b/frame_metadata/common/messages.h @@ -0,0 +1,25 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace frame_metadata { + +struct SensorReading { + double temperature_c = 0.0; +}; + +} // namespace frame_metadata diff --git a/user_timestamped_video/CMakeLists.txt b/frame_metadata/consumer/CMakeLists.txt similarity index 72% rename from user_timestamped_video/CMakeLists.txt rename to frame_metadata/consumer/CMakeLists.txt index 6fdcdd8..add7e16 100644 --- a/user_timestamped_video/CMakeLists.txt +++ b/frame_metadata/consumer/CMakeLists.txt @@ -12,5 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_subdirectory(producer) -add_subdirectory(consumer) +add_executable(frame_metadata_consumer + main.cpp +) + +target_link_libraries(frame_metadata_consumer PRIVATE frame_metadata_support ${LIVEKIT_CORE_TARGET}) + +livekit_copy_windows_runtime_dlls(frame_metadata_consumer) diff --git a/user_timestamped_video/consumer/main.cpp b/frame_metadata/consumer/main.cpp similarity index 56% rename from user_timestamped_video/consumer/main.cpp rename to frame_metadata/consumer/main.cpp index 6ba5d32..2801132 100644 --- a/user_timestamped_video/consumer/main.cpp +++ b/frame_metadata/consumer/main.cpp @@ -14,49 +14,55 @@ * limitations under the License. */ -/// UserTimestampedVideoConsumer +/// frame_metadata_consumer /// /// Receives remote video frames via `Room::setOnVideoFrameEventCallback()` and -/// logs any `VideoFrameMetadata::user_timestamp_us` values that arrive. Pair -/// with `UserTimestampedVideoProducer` running in another process. +/// logs `VideoFrameMetadata::user_timestamp_us`, `frame_id`, and `user_data`. /// /// Usage: -/// UserTimestampedVideoConsumer -/// [--with-user-timestamp|--without-user-timestamp] +/// frame_metadata_consumer [ ] /// -/// Or via environment variables: -/// LIVEKIT_URL, LIVEKIT_TOKEN +/// Or via environment variables (LIVEKIT_URL defaults to ws://localhost:7880): +/// export LIVEKIT_TOKEN= +/// frame_metadata_consumer #include #include +#include #include #include #include +#include #include #include #include -#include "../common/cli_utils.h" +#include "cli_utils.h" +#include "constants.h" +#include "json_converters.h" #include "livekit/livekit.h" +#include "messages.h" using namespace livekit; namespace { -constexpr const char* kTrackName = "timestamped-camera"; - -std::string formatUserTimestamp(const std::optional& metadata) { - if (!metadata || !metadata->user_timestamp_us.has_value()) { - return "n/a"; +std::optional parseSensorReading( + const std::optional& metadata) { + if (!metadata || !metadata->user_data.has_value()) { + return std::nullopt; } - return std::to_string(*metadata->user_timestamp_us); + try { + return frame_metadata::sensorReadingFromJson(frame_metadata::toString(*metadata->user_data)); + } catch (const std::exception&) { + return std::nullopt; + } } -class UserTimestampedVideoConsumerDelegate : public RoomDelegate { +class FrameMetadataConsumerDelegate : public RoomDelegate { public: - UserTimestampedVideoConsumerDelegate(Room& room, bool read_user_timestamp) - : room_(room), read_user_timestamp_(read_user_timestamp) {} + explicit FrameMetadataConsumerDelegate(Room& room) : room_(room) {} void registerExistingParticipants() { for (const auto& weak_participant : room_.remoteParticipants()) { @@ -83,7 +89,7 @@ class UserTimestampedVideoConsumerDelegate : public RoomDelegate { } const std::string identity = event.participant->identity(); - room_.clearOnVideoFrameCallback(identity, std::string(kTrackName)); + room_.clearOnVideoFrameCallback(identity, std::string(frame_metadata::kVideoTrackName)); { std::lock_guard lock(mutex_); @@ -105,32 +111,37 @@ class UserTimestampedVideoConsumerDelegate : public RoomDelegate { VideoStream::Options stream_options; stream_options.format = VideoBufferType::RGBA; - if (read_user_timestamp_) { - room_.setOnVideoFrameEventCallback( - identity, std::string(kTrackName), - [identity](const VideoFrameEvent& event) { - std::cout << "[consumer] from=" << identity << " size=" << event.frame.width() << "x" - << event.frame.height() << " capture_ts_us=" << event.timestamp_us - << " user_ts_us=" << formatUserTimestamp(event.metadata) - << " rotation=" << static_cast(event.rotation) << "\n"; - }, - stream_options); - } else { - room_.setOnVideoFrameCallback( - identity, std::string(kTrackName), - [identity](const VideoFrame& frame, const std::int64_t timestamp_us) { - std::cout << "[consumer] from=" << identity << " size=" << frame.width() << "x" << frame.height() - << " capture_ts_us=" << timestamp_us << " user_ts_us=ignored\n"; - }, - stream_options); - } - - std::cout << "[consumer] listening for video frames from " << identity << " track=\"" << kTrackName - << "\" with user timestamp " << (read_user_timestamp_ ? "enabled" : "ignored") << "\n"; + room_.setOnVideoFrameEventCallback( + identity, std::string(frame_metadata::kVideoTrackName), + [](const VideoFrameEvent& event) { + if (!event.metadata || !event.metadata->frame_id || !event.metadata->user_timestamp_us || + !event.metadata->user_data) { + return; + } + + const std::uint32_t frame_id = *event.metadata->frame_id; + if (frame_id % 5 != 0) { + return; + } + + const auto reading = parseSensorReading(event.metadata); + if (!reading) { + return; + } + + std::cout << std::fixed << std::setprecision(2) << "[consumer] frame_id=" << frame_id + << " capture_ts_us=" << event.timestamp_us + << " user_ts_us=" << *event.metadata->user_timestamp_us + << " temperature_c=" << reading->temperature_c + << " user_data_bytes=" << event.metadata->user_data->size() << "\n"; + }, + stream_options); + + std::cout << "[consumer] listening for video frames from " << identity << " track=\"" + << frame_metadata::kVideoTrackName << "\" with frame metadata\n"; } Room& room_; - bool read_user_timestamp_; std::mutex mutex_; std::unordered_set registered_identities_; }; @@ -138,15 +149,15 @@ class UserTimestampedVideoConsumerDelegate : public RoomDelegate { } // namespace int main(int argc, char* argv[]) { - user_timestamped_video::CliOptions cli_options; + frame_metadata::CliOptions cli_options; - const user_timestamped_video::ParseResult parse_result = user_timestamped_video::parseArgs(argc, argv, cli_options); - if (parse_result != user_timestamped_video::ParseResult::Ok) { - user_timestamped_video::printUsage(argv[0]); - return parse_result == user_timestamped_video::ParseResult::Help ? 0 : 1; + const frame_metadata::ParseResult parse_result = frame_metadata::parseArgs(argc, argv, cli_options); + if (parse_result != frame_metadata::ParseResult::Ok) { + frame_metadata::printUsage(argv[0]); + return parse_result == frame_metadata::ParseResult::Help ? 0 : 1; } - user_timestamped_video::installSignalHandlers(); + frame_metadata::installSignalHandlers(); livekit::initialize(livekit::LogLevel::Info); int exit_code = 0; @@ -157,7 +168,7 @@ int main(int argc, char* argv[]) { options.auto_subscribe = true; options.dynacast = false; - UserTimestampedVideoConsumerDelegate delegate(room, cli_options.use_user_timestamp); + FrameMetadataConsumerDelegate delegate(room); room.setDelegate(&delegate); std::cout << "[consumer] connecting to " << cli_options.url << "\n"; @@ -167,21 +178,20 @@ int main(int argc, char* argv[]) { } else { if (auto lp = room.localParticipant().lock()) { std::cout << "[consumer] connected as " << (lp ? lp->identity() : std::string("")) << " to room '" - << room.roomInfo().name << "' with user timestamp " - << (cli_options.use_user_timestamp ? "enabled" : "ignored") << "\n"; + << room.roomInfo().name << "'\n"; } else { throw std::runtime_error("unable to lock local participant"); } delegate.registerExistingParticipants(); - while (user_timestamped_video::isRunning()) { + while (frame_metadata::isRunning()) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } for (const auto& weak_participant : room.remoteParticipants()) { if (auto participant = weak_participant.lock()) { - room.clearOnVideoFrameCallback(participant->identity(), std::string(kTrackName)); + room.clearOnVideoFrameCallback(participant->identity(), std::string(frame_metadata::kVideoTrackName)); } else { throw std::runtime_error("unable to lock provided remote participant"); } diff --git a/user_timestamped_video/producer/CMakeLists.txt b/frame_metadata/producer/CMakeLists.txt similarity index 67% rename from user_timestamped_video/producer/CMakeLists.txt rename to frame_metadata/producer/CMakeLists.txt index 5c573ea..934af55 100644 --- a/user_timestamped_video/producer/CMakeLists.txt +++ b/frame_metadata/producer/CMakeLists.txt @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_executable(UserTimestampedVideoProducer +add_executable(frame_metadata_producer main.cpp ) -target_include_directories(UserTimestampedVideoProducer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(UserTimestampedVideoProducer PRIVATE ${LIVEKIT_CORE_TARGET}) +target_link_libraries(frame_metadata_producer PRIVATE frame_metadata_support ${LIVEKIT_CORE_TARGET}) -livekit_copy_windows_runtime_dlls(UserTimestampedVideoProducer) +livekit_copy_windows_runtime_dlls(frame_metadata_producer) diff --git a/user_timestamped_video/producer/main.cpp b/frame_metadata/producer/main.cpp similarity index 55% rename from user_timestamped_video/producer/main.cpp rename to frame_metadata/producer/main.cpp index c58a983..1232547 100644 --- a/user_timestamped_video/producer/main.cpp +++ b/frame_metadata/producer/main.cpp @@ -14,29 +14,34 @@ * limitations under the License. */ -/// UserTimestampedVideoProducer +/// frame_metadata_producer /// -/// Publishes a synthetic camera track and stamps each frame with -/// `VideoCaptureOptions::metadata.user_timestamp_us`. Pair with -/// `UserTimestampedVideoConsumer` in another process to observe the user -/// timestamps flowing end to end. +/// Publishes a synthetic camera track and stamps each frame with v1.3.0 frame +/// metadata: `user_timestamp_us`, `frame_id`, and compact `user_data`. /// /// Usage: -/// UserTimestampedVideoProducer -/// [--with-user-timestamp|--without-user-timestamp] +/// frame_metadata_producer [ ] /// -/// Or via environment variables: -/// LIVEKIT_URL, LIVEKIT_TOKEN +/// Or via environment variables (LIVEKIT_URL defaults to ws://localhost:7880): +/// export LIVEKIT_TOKEN= +/// frame_metadata_producer +#include #include +#include #include +#include #include #include +#include #include #include -#include "../common/cli_utils.h" +#include "cli_utils.h" +#include "constants.h" +#include "json_converters.h" #include "livekit/livekit.h" +#include "messages.h" using namespace livekit; @@ -52,10 +57,15 @@ std::uint64_t nowEpochUs() { .count()); } -void fillFrame(VideoFrame& frame, std::uint32_t frame_index) { - const std::uint8_t blue = static_cast((frame_index * 7) % 255); - const std::uint8_t green = static_cast((frame_index * 13) % 255); - const std::uint8_t red = static_cast((frame_index * 29) % 255); +double simulatedTemperatureC(std::uint32_t frame_id) { + return 22.0 + (2.5 * std::sin(static_cast(frame_id) * 0.15)); +} + +void fillFrame(VideoFrame& frame, double temperature_c) { + const double normalized = std::max(0.0, std::min(1.0, (temperature_c - 18.0) / 10.0)); + const auto red = static_cast(80.0 + (normalized * 175.0)); + const auto blue = static_cast(255.0 - (normalized * 120.0)); + const auto green = static_cast(90); std::uint8_t* data = frame.data(); for (std::size_t i = 0; i < frame.dataSize(); i += 4) { @@ -69,15 +79,15 @@ void fillFrame(VideoFrame& frame, std::uint32_t frame_index) { } // namespace int main(int argc, char* argv[]) { - user_timestamped_video::CliOptions cli_options; + frame_metadata::CliOptions cli_options; - const user_timestamped_video::ParseResult parse_result = user_timestamped_video::parseArgs(argc, argv, cli_options); - if (parse_result != user_timestamped_video::ParseResult::Ok) { - user_timestamped_video::printUsage(argv[0]); - return parse_result == user_timestamped_video::ParseResult::Help ? 0 : 1; + const frame_metadata::ParseResult parse_result = frame_metadata::parseArgs(argc, argv, cli_options); + if (parse_result != frame_metadata::ParseResult::Ok) { + frame_metadata::printUsage(argv[0]); + return parse_result == frame_metadata::ParseResult::Help ? 0 : 1; } - user_timestamped_video::installSignalHandlers(); + frame_metadata::installSignalHandlers(); livekit::initialize(livekit::LogLevel::Info); int exit_code = 0; @@ -100,52 +110,57 @@ int main(int argc, char* argv[]) { } auto source = std::make_shared(kFrameWidth, kFrameHeight); - auto track = LocalVideoTrack::createLocalVideoTrack("timestamped-camera", source); + auto track = LocalVideoTrack::createLocalVideoTrack(frame_metadata::kVideoTrackName, source); try { TrackPublishOptions publish_options; publish_options.source = TrackSource::SOURCE_CAMERA; - publish_options.packet_trailer_features.user_timestamp = cli_options.use_user_timestamp; + publish_options.frame_metadata_features = FrameMetadataFeatures{}; + publish_options.frame_metadata_features->user_timestamp = true; + publish_options.frame_metadata_features->frame_id = true; + publish_options.frame_metadata_features->user_data = true; { auto lp = room.localParticipant().lock(); if (!lp) throw std::runtime_error("local participant unavailable"); lp->publishTrack(track, publish_options); - std::cout << "[producer] published camera track with user timestamp " - << (cli_options.use_user_timestamp ? "enabled" : "disabled") << "\n"; + std::cout << "[producer] published camera track \"" << frame_metadata::kVideoTrackName + << "\" with frame metadata\n"; } VideoFrame frame = VideoFrame::create(kFrameWidth, kFrameHeight, VideoBufferType::BGRA); const auto capture_start = std::chrono::steady_clock::now(); - std::uint32_t frame_index = 0; + std::uint32_t frame_id = 0; auto next_frame_at = std::chrono::steady_clock::now(); - while (user_timestamped_video::isRunning()) { - fillFrame(frame, frame_index); + while (frame_metadata::isRunning()) { + const std::uint64_t user_timestamp_us = nowEpochUs(); + const double temperature_c = simulatedTemperatureC(frame_id); - VideoCaptureOptions capture_options; + fillFrame(frame, temperature_c); - // a steady_clock to align with other data/video frames + VideoCaptureOptions capture_options; capture_options.timestamp_us = static_cast( std::chrono::duration_cast(std::chrono::steady_clock::now() - capture_start) .count()); capture_options.rotation = VideoRotation::VIDEO_ROTATION_0; - if (cli_options.use_user_timestamp) { - capture_options.metadata = VideoFrameMetadata{}; - capture_options.metadata->user_timestamp_us = nowEpochUs(); - } + + capture_options.metadata = VideoFrameMetadata{}; + capture_options.metadata->user_timestamp_us = user_timestamp_us; + capture_options.metadata->frame_id = frame_id; + capture_options.metadata->user_data = frame_metadata::toPayload( + frame_metadata::sensorReadingToJson(frame_metadata::SensorReading{temperature_c})); source->captureFrame(frame, capture_options); - if (frame_index % 5 == 0) { - std::cout << "[producer] frame=" << frame_index << " capture_ts_us=" << capture_options.timestamp_us - << " user_ts_us=" - << (cli_options.use_user_timestamp ? std::to_string(*capture_options.metadata->user_timestamp_us) - : std::string("disabled")) - << "\n"; + if (frame_id % 5 == 0) { + std::cout << std::fixed << std::setprecision(2) << "[producer] frame_id=" << frame_id + << " capture_ts_us=" << capture_options.timestamp_us + << " user_ts_us=" << user_timestamp_us << " temperature_c=" << temperature_c + << " user_data_bytes=" << capture_options.metadata->user_data->size() << "\n"; } - ++frame_index; + ++frame_id; next_frame_at += std::chrono::milliseconds(kFrameIntervalMs); std::this_thread::sleep_until(next_frame_at); } @@ -154,7 +169,6 @@ int main(int argc, char* argv[]) { exit_code = 1; } - // best effort unpublish if (auto lp = room.localParticipant().lock()) { if (lp && track->publication()) { lp->unpublishTrack(track->publication()->sid()); diff --git a/user_timestamped_video/README.md b/user_timestamped_video/README.md deleted file mode 100644 index ea9fd27..0000000 --- a/user_timestamped_video/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# UserTimestampedVideo - -This example is split into two executables and can demonstrate all four -producer/consumer combinations: - -- `UserTimestampedVideoProducer` publishes a synthetic video track named - `"timestamped-camera"` and stamps each frame with - `VideoCaptureOptions::metadata.user_timestamp_us`. -- `UserTimestampedVideoConsumer` subscribes to the remote - `"timestamped-camera"` track by name with either the rich or legacy callback - path. - -Run them in the same room with different participant identities: - -```sh -LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoProducer -LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoConsumer -``` - -Requirements: - -- LiveKit C++ SDK `v0.3.4` or newer. This example uses - `VideoFrameMetadata` and `setOnVideoFrameEventCallback`, which are not - available in older SDK releases. -- To pin the SDK version when configuring the examples, pass - `-DLIVEKIT_SDK_VERSION=0.3.4` to CMake. - -Flags: - -- Producer default: sends user timestamps -- Producer `--with-user-timestamp`: explicitly sends user timestamps -- Producer `--without-user-timestamp`: does not send user timestamps -- Consumer default: reads user timestamps through `setOnVideoFrameEventCallback` -- Consumer `--with-user-timestamp`: explicitly reads user timestamps through - `setOnVideoFrameEventCallback` -- Consumer `--without-user-timestamp`: ignores metadata through the legacy - `setOnVideoFrameCallback` - -Matrix: - -```sh -# 1. Producer sends, consumer reads -./UserTimestampedVideoProducer -./UserTimestampedVideoConsumer - -# 2. Producer sends, consumer ignores -./UserTimestampedVideoProducer -./UserTimestampedVideoConsumer --without-user-timestamp - -# 3. Producer does not send, consumer ignores -./UserTimestampedVideoProducer --without-user-timestamp -./UserTimestampedVideoConsumer --without-user-timestamp - -# 4. Producer does not send, consumer reads -./UserTimestampedVideoProducer --without-user-timestamp -./UserTimestampedVideoConsumer -``` - -Timestamp note: - -- `user_ts_us` is application metadata and is the value to compare end to end. -- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`. -- `capture_ts_us` on the consumer is the received WebRTC frame timestamp. -- Producer and consumer `capture_ts_us` values are not expected to match exactly, - because WebRTC may translate frame timestamps onto its own internal - capture-time timeline before delivery.