Frame metadata example

CMakeLists.txt

@@ -92,4 +92,4 @@ add_subdirectory(hello_livekit/receiver hello_livekit_receiver)
 add_subdirectory(platform_audio)
 add_subdirectory(ping_pong/ping ping_pong_ping)
 add_subdirectory(ping_pong/pong ping_pong_pong)
-add_subdirectory(user_timestamped_video)
+add_subdirectory(frame_metadata)

README.md

@@ -73,16 +73,6 @@ For example:
 ./build/basic_room/basic_room --url <ws-url> --token <token>
 ```
 
-### PlatformAudio
-
-The `platform_audio` examples show microphone capture and speaker playout using
-WebRTC's platform Audio Device Module:
-
-```bash
-./build/platform_audio/player/PlatformAudioPlayer <ws-url> <player-token>
-./build/platform_audio/sender/PlatformAudioSender <ws-url> <sender-token>
-```
-
 ### Supported platforms
 
 Prebuilt SDKs are downloaded automatically for:

user_timestamped_video/consumer/CMakeLists.txt → frame_metadata/CMakeLists.txt

@@ -12,11 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-add_executable(UserTimestampedVideoConsumer
-  main.cpp
+add_library(frame_metadata_support STATIC
+  common/json_converters.cpp
+  common/json_converters.h
+  common/cli_utils.h
+  common/constants.h
+  common/messages.h
 )
 
-target_include_directories(UserTimestampedVideoConsumer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-target_link_libraries(UserTimestampedVideoConsumer PRIVATE ${LIVEKIT_CORE_TARGET})
+target_include_directories(frame_metadata_support PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/common)
+target_link_libraries(frame_metadata_support PUBLIC nlohmann_json::nlohmann_json)
 
-livekit_copy_windows_runtime_dlls(UserTimestampedVideoConsumer)
+add_subdirectory(producer)
+add_subdirectory(consumer)

frame_metadata/README.md

@@ -0,0 +1,50 @@
+# Frame Metadata Example
+
+This example is split into two executables and demonstrates LiveKit C++ SDK video frame metadata feature:
+
+- `frame_metadata_producer` publishes a synthetic video track named `"sensor-camera"`.
+- Each video frame attaches additional data via the frame metadata features of the SDK:
+
+  | Field | Description |
+  | --- | --- |
+  | `user_timestamp_us` | Wall-clock capture time (us) |
+  | `frame_id` | Monotonic frame counter |
+  | `user_data` | Free-form user data, in this example it is used to send a JSON temperature reading correlated with the video frame |
+
+- `frame_metadata_consumer` subscribes to the remote track by name and reads the
+  metadata through `setOnVideoFrameEventCallback`.
+
+Run them in the same room with different participant identities. `LIVEKIT_URL`
+defaults to `ws://localhost:7880` when unset:
+
+```sh
+export LIVEKIT_TOKEN=<producer-token>
+./build/frame_metadata/producer/frame_metadata_producer
+
+export LIVEKIT_TOKEN=<consumer-token>
+./build/frame_metadata/consumer/frame_metadata_consumer
+```
+
+You can also pass URL and token explicitly:
+
+```sh
+./build/frame_metadata/producer/frame_metadata_producer ws://localhost:7880 <producer-token>
+./build/frame_metadata/consumer/frame_metadata_consumer ws://localhost:7880 <consumer-token>
+```
+
+Metadata notes for this example:
+
+- `frame_id` and `user_ts_us` come from `VideoFrameMetadata` and are the values
+  to compare end to end.
+- `user_data` is free-form frame metadata. This example uses it only for the
+  temperature reading, similar to a robotics or sensor-fusion camera feed, in the following format:
+
+```json
+{"temperature_c":23.4}
+```
+
+- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`.
+- `capture_ts_us` on the consumer is the received WebRTC frame timestamp.
+- Producer and consumer `capture_ts_us` values are not expected to match exactly,
+  because WebRTC may translate frame timestamps onto its own internal
+  capture-time timeline before delivery.

user_timestamped_video/common/cli_utils.h → frame_metadata/common/cli_utils.h

@@ -18,19 +18,21 @@
 
 #include <atomic>
 #include <csignal>
+#include <cstdint>
 #include <cstdlib>
 #include <iostream>
 #include <string>
 #include <vector>
 
-namespace user_timestamped_video {
+namespace frame_metadata {
+
+inline constexpr char kDefaultLiveKitUrl[] = "ws://localhost:7880";
 
 enum class ParseResult { Ok, Help, Error };
 
 struct CliOptions {
   std::string url;
   std::string token;
-  bool use_user_timestamp = true;
 };
 
 inline std::atomic<bool> g_running{true};
@@ -53,11 +55,15 @@ inline std::string getenvOrEmpty(const char* name) {
 
 inline void printUsage(const char* program) {
   std::cerr << "Usage:\n"
-            << "  " << program << " <ws-url> <token> "
-            << "[--with-user-timestamp|--without-user-timestamp]\n"
-            << "or:\n"
-            << "  LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program
-            << " [--with-user-timestamp|--without-user-timestamp]\n";
+            << "  " << program << " [<ws-url> <token>]\n"
+            << "\n"
+            << "Environment:\n"
+            << "  LIVEKIT_URL    defaults to " << kDefaultLiveKitUrl << " when unset\n"
+            << "  LIVEKIT_TOKEN  required unless passed as the second argument\n"
+            << "\n"
+            << "Example:\n"
+            << "  export LIVEKIT_TOKEN=<token>\n"
+            << "  " << program << "\n";
 }
 
 inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) {
@@ -69,14 +75,6 @@ inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) {
     if (arg == "-h" || arg == "--help") {
       return ParseResult::Help;
     }
-    if (arg == "--without-user-timestamp") {
-      options.use_user_timestamp = false;
-      continue;
-    }
-    if (arg == "--with-user-timestamp") {
-      options.use_user_timestamp = true;
-      continue;
-    }
     if (!arg.empty() && arg[0] == '-') {
       return ParseResult::Error;
     }
@@ -98,7 +96,19 @@ inline ParseResult parseArgs(int argc, char* argv[], CliOptions& options) {
     return ParseResult::Error;
   }
 
-  return (options.url.empty() || options.token.empty()) ? ParseResult::Error : ParseResult::Ok;
+  if (options.url.empty()) {
+    options.url = kDefaultLiveKitUrl;
+  }
+
+  return options.token.empty() ? ParseResult::Error : ParseResult::Ok;
+}
+
+inline std::vector<std::uint8_t> toPayload(const std::string& text) {
+  return std::vector<std::uint8_t>(text.begin(), text.end());
+}
+
+inline std::string toString(const std::vector<std::uint8_t>& payload) {
+  return std::string(payload.begin(), payload.end());
 }
 
-} // namespace user_timestamped_video
+} // namespace frame_metadata

frame_metadata/common/constants.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace frame_metadata {
+
+inline constexpr char kVideoTrackName[] = "sensor-camera";
+
+inline constexpr char kTemperatureCKey[] = "temperature_c";
+
+} // namespace frame_metadata

frame_metadata/common/json_converters.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "json_converters.h"
+
+#include <nlohmann/json.hpp>
+
+#include "constants.h"
+
+namespace frame_metadata {
+
+std::string sensorReadingToJson(const SensorReading& reading) {
+  nlohmann::json json;
+  json[kTemperatureCKey] = reading.temperature_c;
+  return json.dump();
+}
+
+std::optional<SensorReading> sensorReadingFromJson(const std::string& json_text) {
+  const auto json = nlohmann::json::parse(json_text, nullptr, false);
+  if (json.is_discarded() || !json.is_object()) {
+    return std::nullopt;
+  }
+
+  const auto temperature_it = json.find(kTemperatureCKey);
+  if (temperature_it == json.end() || !temperature_it->is_number()) {
+    return std::nullopt;
+  }
+
+  SensorReading reading;
+  reading.temperature_c = temperature_it->get<double>();
+  return reading;
+}
+
+} // namespace frame_metadata

frame_metadata/common/json_converters.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <optional>
+#include <string>
+
+#include "messages.h"
+
+namespace frame_metadata {
+
+std::string sensorReadingToJson(const SensorReading& reading);
+std::optional<SensorReading> sensorReadingFromJson(const std::string& json_text);
+
+} // namespace frame_metadata

frame_metadata/common/messages.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace frame_metadata {
+
+struct SensorReading {
+  double temperature_c = 0.0;
+};
+
+} // namespace frame_metadata

user_timestamped_video/CMakeLists.txt → frame_metadata/consumer/CMakeLists.txt

@@ -12,5 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-add_subdirectory(producer)
-add_subdirectory(consumer)
+add_executable(frame_metadata_consumer
+  main.cpp
+)
+
+target_link_libraries(frame_metadata_consumer PRIVATE frame_metadata_support ${LIVEKIT_CORE_TARGET})
+
+livekit_copy_windows_runtime_dlls(frame_metadata_consumer)