From 882c01fce0d424d05a19a76eb287a5eedfd0b512 Mon Sep 17 00:00:00 2001 From: daniel-g-favoreto-opl <112583028+daniel-g-favoreto-opl@users.noreply.github.com> Date: Mon, 28 Jul 2025 05:01:36 -0300 Subject: [PATCH 1/3] RECORDINGS - Add fallback resolutions for unsupported stream frame sizes on low-end Android devices (#1900) * add fallback resolutions * add missing imports --- .../webrtc/record/VideoFileRenderer.java | 94 ++++++++++++------- 1 file changed, 62 insertions(+), 32 deletions(-) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/record/VideoFileRenderer.java b/android/src/main/java/com/cloudwebrtc/webrtc/record/VideoFileRenderer.java index 275b33264a..b1dbd08f51 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/record/VideoFileRenderer.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/record/VideoFileRenderer.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.CountDownLatch; class VideoFileRenderer implements VideoSink, SamplesReadyCallback { @@ -78,39 +81,66 @@ class VideoFileRenderer implements VideoSink, SamplesReadyCallback { audioTrackIndex = withAudio ? -1 : 0; } - private void initVideoEncoder() { - MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, outputFileWidth, outputFileHeight); - - // Set some properties. Failing to specify some of these can cause the MediaCodec - // configure() call to throw an unhelpful exception. - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, - MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); - format.setInteger(MediaFormat.KEY_BIT_RATE, 6000000); - format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE); - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL); - - format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 0); // Para Surface input - format.setInteger(MediaFormat.KEY_PRIORITY, 0); // Background priority - format.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline); // AVC baseline - - // Create a MediaCodec encoder, and configure it with our format. Get a Surface - // we can use for input and wrap it with a class that handles the EGL work. + private boolean tryConfigureEncoder(int width, int height) { try { + MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, width, height); + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, + MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + format.setInteger(MediaFormat.KEY_BIT_RATE, 6000000); + format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE); + format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL); + format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 0); + format.setInteger(MediaFormat.KEY_PRIORITY, 0); + format.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline); + encoder = MediaCodec.createEncoderByType(MIME_TYPE); encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - - CountDownLatch latch = new CountDownLatch(1); - renderThreadHandler.post(() -> { - eglBase = EglBase.create(sharedContext, EglBase.CONFIG_RECORDABLE); - surface = encoder.createInputSurface(); - eglBase.createSurface(surface); - eglBase.makeCurrent(); - drawer = new GlRectDrawer(); - latch.countDown(); - }); - latch.await(); // espera EGL estar pronto + outputFileWidth = width; + outputFileHeight = height; + return true; } catch (Exception e) { - Log.wtf(TAG, e); + if (encoder != null) { + try { encoder.release(); } catch (Exception ignored) {} + encoder = null; + } + return false; + } + } + + private void initVideoEncoder(int frameWidth, int frameHeight) { + List resolutions = new ArrayList<>(); + resolutions.add(new int[]{frameWidth, frameHeight}); + resolutions.addAll(Arrays.asList( + new int[]{1920, 1080}, + new int[]{1280, 720}, + new int[]{854, 480}, + new int[]{640, 360}, + new int[]{426, 240} + )); + + for (int[] res : resolutions) { + if (tryConfigureEncoder(res[0], res[1])) { + break; + } + } + + if (encoder == null) { + Log.e(TAG, "Failed to configure encoder with any supported resolution."); + return; + } + + CountDownLatch latch = new CountDownLatch(1); + renderThreadHandler.post(() -> { + eglBase = EglBase.create(sharedContext, EglBase.CONFIG_RECORDABLE); + surface = encoder.createInputSurface(); + eglBase.createSurface(surface); + eglBase.makeCurrent(); + drawer = new GlRectDrawer(); + latch.countDown(); + }); + try { + latch.await(); + } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } @@ -119,9 +149,9 @@ private void initVideoEncoder() { public void onFrame(VideoFrame frame) { frame.retain(); if (outputFileWidth == -1) { - outputFileWidth = frame.getRotatedWidth(); - outputFileHeight = frame.getRotatedHeight(); - initVideoEncoder(); + int frameWidth = frame.getRotatedWidth(); + int frameHeight = frame.getRotatedHeight(); + initVideoEncoder(frameWidth, frameHeight); } renderThreadHandler.post(() -> renderFrameOnRenderThread(frame)); } From 2eb041d44147346919e95fa564181209fd3c53de Mon Sep 17 00:00:00 2001 From: Gautam Tirkha <82673815+gktirkha@users.noreply.github.com> Date: Mon, 28 Jul 2025 13:31:49 +0530 Subject: [PATCH 2/3] Update proguard-rules.pro (#1902) --- android/proguard-rules.pro | 1 + 1 file changed, 1 insertion(+) diff --git a/android/proguard-rules.pro b/android/proguard-rules.pro index 6ce9896196..699b363c62 100644 --- a/android/proguard-rules.pro +++ b/android/proguard-rules.pro @@ -1,3 +1,4 @@ # Flutter WebRTC -keep class com.cloudwebrtc.webrtc.** { *; } -keep class org.webrtc.** { *; } +-keep class org.jni_zero.** { *; } From b68a73cfcc7a3b5efa9ce27c42a99750b97cac5c Mon Sep 17 00:00:00 2001 From: David Chen Date: Wed, 6 Aug 2025 21:55:08 -0700 Subject: [PATCH 3/3] feat: Add texture-based video rendering for web (#1911) --- lib/src/web/rtc_video_renderer_impl.dart | 149 ++++++++++++-------- lib/src/web/rtc_video_view_impl.dart | 169 ++++++++++++++++++++++- 2 files changed, 256 insertions(+), 62 deletions(-) diff --git a/lib/src/web/rtc_video_renderer_impl.dart b/lib/src/web/rtc_video_renderer_impl.dart index 69df097e0c..109abca3d7 100644 --- a/lib/src/web/rtc_video_renderer_impl.dart +++ b/lib/src/web/rtc_video_renderer_impl.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; import 'dart:ui_web' as web_ui; import 'package:flutter/foundation.dart'; @@ -8,6 +9,9 @@ import 'package:flutter/services.dart'; import 'package:dart_webrtc/dart_webrtc.dart'; import 'package:web/web.dart' as web; +const bool useHtmlElementView = + bool.fromEnvironment("WEBRTC_USE_HTML_ELEMENT_VIEW", defaultValue: false); + // An error code value to error name Map. // See: https://developer.mozilla.org/en-US/docs/Web/API/MediaError/code const Map _kErrorValueToErrorName = { @@ -59,6 +63,8 @@ class RTCVideoRenderer extends ValueNotifier bool _muted = false; + web.HTMLVideoElement? element; + set objectFit(String fit) { if (_objectFit == fit) return; _objectFit = fit; @@ -233,6 +239,9 @@ class RTCVideoRenderer extends ValueNotifier if (audioManager != null && !audioManager.hasChildNodes()) { audioManager.remove(); } + if (!useHtmlElementView) { + element?.remove(); + } return super.dispose(); } @@ -240,8 +249,11 @@ class RTCVideoRenderer extends ValueNotifier Future audioOutput(String deviceId) async { try { final element = _audioElement; - if (null != element) { - await element.setSinkId(deviceId).toDart; + if (null != element && + element.getProperty('setSinkId'.toJS).isDefinedAndNotNull) { + await (element.callMethod('setSinkId'.toJS, deviceId.toJS) as JSPromise) + .toDart; + return true; } } catch (e) { @@ -250,62 +262,71 @@ class RTCVideoRenderer extends ValueNotifier return false; } + web.HTMLVideoElement createElement() { + for (var s in _subscriptions) { + s.cancel(); + } + _subscriptions.clear(); + + final element = web.HTMLVideoElement() + ..autoplay = true + ..muted = true + ..controls = false + ..srcObject = _videoStream + ..id = _elementIdForVideo + ..setAttribute('playsinline', 'true'); + + _applyDefaultVideoStyles(element); + + _subscriptions.add( + element.onCanPlay.listen((dynamic _) { + _updateAllValues(element); + }), + ); + + _subscriptions.add( + element.onResize.listen((dynamic _) { + _updateAllValues(element); + onResize?.call(); + }), + ); + + // The error event fires when some form of error occurs while attempting to load or perform the media. + _subscriptions.add( + element.onError.listen((web.Event _) { + // The Event itself (_) doesn't contain info about the actual error. + // We need to look at the HTMLMediaElement.error. + // See: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/error + final error = element.error; + print('RTCVideoRenderer: videoElement.onError, ${error.toString()}'); + throw PlatformException( + code: _kErrorValueToErrorName[error!.code]!, + message: error.message != '' ? error.message : _kDefaultErrorMessage, + details: _kErrorValueToErrorDescription[error.code], + ); + }), + ); + + _subscriptions.add( + element.onEnded.listen((dynamic _) { + // print('RTCVideoRenderer: videoElement.onEnded'); + }), + ); + + return element; + } + @override Future initialize() async { - web_ui.platformViewRegistry.registerViewFactory(viewType, (int viewId) { - for (var s in _subscriptions) { - s.cancel(); - } - _subscriptions.clear(); - - final element = web.HTMLVideoElement() - ..autoplay = true - ..muted = true - ..controls = false - ..srcObject = _videoStream - ..id = _elementIdForVideo - ..setAttribute('playsinline', 'true'); - - _applyDefaultVideoStyles(element); - - _subscriptions.add( - element.onCanPlay.listen((dynamic _) { - _updateAllValues(element); - }), - ); - - _subscriptions.add( - element.onResize.listen((dynamic _) { - _updateAllValues(element); - onResize?.call(); - }), - ); - - // The error event fires when some form of error occurs while attempting to load or perform the media. - _subscriptions.add( - element.onError.listen((web.Event _) { - // The Event itself (_) doesn't contain info about the actual error. - // We need to look at the HTMLMediaElement.error. - // See: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/error - final error = element.error; - print('RTCVideoRenderer: videoElement.onError, ${error.toString()}'); - throw PlatformException( - code: _kErrorValueToErrorName[error!.code]!, - message: - error.message != '' ? error.message : _kDefaultErrorMessage, - details: _kErrorValueToErrorDescription[error.code], - ); - }), - ); - - _subscriptions.add( - element.onEnded.listen((dynamic _) { - // print('RTCVideoRenderer: videoElement.onEnded'); - }), - ); - - return element; - }); + bool isVisible = useHtmlElementView; + if (isVisible) { + web_ui.platformViewRegistry.registerViewFactory(viewType, (int viewId) { + return createElement(); + }, isVisible: isVisible); + } else { + final element = createElement(); + web.window.document.body!.appendChild(element); + } } void _applyDefaultVideoStyles(web.HTMLVideoElement element) { @@ -314,11 +335,17 @@ class RTCVideoRenderer extends ValueNotifier element.style.transform = 'scaleX(-1)'; } - element - ..style.objectFit = _objectFit - ..style.border = 'none' - ..style.width = '100%' - ..style.height = '100%'; + if (useHtmlElementView) { + element + ..style.objectFit = _objectFit + ..style.border = 'none' + ..style.width = '100%' + ..style.height = '100%'; + } else { + element.style.pointerEvents = "none"; + element.style.opacity = "0"; + element.style.position = "absolute"; + } } @override diff --git a/lib/src/web/rtc_video_view_impl.dart b/lib/src/web/rtc_video_view_impl.dart index 9ef8ff1461..fab26ca0b8 100644 --- a/lib/src/web/rtc_video_view_impl.dart +++ b/lib/src/web/rtc_video_view_impl.dart @@ -1,8 +1,13 @@ import 'dart:async'; +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; +import 'dart:ui' as ui; +import 'dart:ui_web' as ui_web; import 'package:flutter/material.dart'; import 'package:dart_webrtc/dart_webrtc.dart'; +import 'package:web/web.dart' as web; import 'package:webrtc_interface/webrtc_interface.dart'; import 'rtc_video_renderer_impl.dart'; @@ -41,17 +46,97 @@ class RTCVideoViewState extends State { widget.objectFit == RTCVideoViewObjectFit.RTCVideoViewObjectFitContain ? 'contain' : 'cover'; + + videoElement = + web.document.getElementById("video_${videoRenderer.viewType}") + as web.HTMLVideoElement?; + frameCallback(0.toJS, 0.toJS); } void _onRendererListener() { if (mounted) setState(() {}); } + int? callbackID; + + void getFrame(web.HTMLVideoElement element) { + callbackID = + element.requestVideoFrameCallbackWithFallback(frameCallback.toJS); + } + + void cancelFrame(web.HTMLVideoElement element) { + if (callbackID != null) { + element.cancelVideoFrameCallbackWithFallback(callbackID!); + } + } + + void frameCallback(JSAny now, JSAny metadata) { + final web.HTMLVideoElement? element = videoElement; + if (element != null) { + // only capture frames if video is playing (optimization for RAF) + if (element.readyState > 2) { + capture().then((_) async { + getFrame(element); + }); + } else { + getFrame(element); + } + } else { + if (mounted) { + Future.delayed(Duration(milliseconds: 100)).then((_) { + frameCallback(0.toJS, 0.toJS); + }); + } + } + } + + ui.Image? capturedFrame; + num? lastFrameTime; + Future capture() async { + final element = videoElement!; + if (lastFrameTime != element.currentTime) { + lastFrameTime = element.currentTime; + try { + final ui.Image img = await ui_web.createImageFromTextureSource(element, + width: element.videoWidth, + height: element.videoHeight, + transferOwnership: true); + + if (mounted) { + setState(() { + capturedFrame?.dispose(); + capturedFrame = img; + }); + } + } on web.DOMException catch (err) { + lastFrameTime = null; + if (err.name == 'InvalidStateError') { + // We don't have enough data yet, continue on + } else { + rethrow; + } + } + } + } + @override void dispose() { if (mounted) { super.dispose(); } + capturedFrame?.dispose(); + if (videoElement != null) { + cancelFrame(videoElement!); + } + } + + Size? size; + + void updateElement() { + if (videoElement != null && size != null) { + videoElement!.width = size!.width.toInt(); + videoElement!.height = size!.height.toInt(); + } } @override @@ -65,8 +150,40 @@ class RTCVideoViewState extends State { : 'cover'; } + web.HTMLVideoElement? videoElement; + Widget buildVideoElementView() { - return HtmlElementView(viewType: videoRenderer.viewType); + if (useHtmlElementView) { + return HtmlElementView(viewType: videoRenderer.viewType); + } else { + return LayoutBuilder(builder: (context, constraints) { + if (videoElement != null && size != constraints.biggest) { + size = constraints.biggest; + updateElement(); + } + + return Stack(children: [ + if (capturedFrame != null) + Positioned.fill( + child: FittedBox( + fit: switch (widget.objectFit) { + RTCVideoViewObjectFit.RTCVideoViewObjectFitContain => + BoxFit.contain, + RTCVideoViewObjectFit.RTCVideoViewObjectFitCover => + BoxFit.cover, + }, + child: SizedBox( + width: capturedFrame!.width.toDouble(), + height: capturedFrame!.height.toDouble(), + child: CustomPaint( + willChange: true, + painter: _ImageFlipPainter( + capturedFrame!, + widget.mirror, + ))))) + ]); + }); + } } @override @@ -86,3 +203,53 @@ class RTCVideoViewState extends State { ); } } + +typedef _VideoFrameRequestCallback = JSFunction; + +extension _HTMLVideoElementRequestAnimationFrame on web.HTMLVideoElement { + int requestVideoFrameCallbackWithFallback( + _VideoFrameRequestCallback callback) { + if (hasProperty('requestVideoFrameCallback'.toJS).toDart) { + return requestVideoFrameCallback(callback); + } else { + return web.window.requestAnimationFrame((double num) { + callback.callAsFunction(this, 0.toJS, 0.toJS); + }.toJS); + } + } + + void cancelVideoFrameCallbackWithFallback(int callbackID) { + if (hasProperty('requestVideoFrameCallback'.toJS).toDart) { + cancelVideoFrameCallback(callbackID); + } else { + web.window.cancelAnimationFrame(callbackID); + } + } + + external int requestVideoFrameCallback(_VideoFrameRequestCallback callback); + external void cancelVideoFrameCallback(int callbackID); +} + +class _ImageFlipPainter extends CustomPainter { + _ImageFlipPainter(this.image, this.flip); + + final ui.Image image; + final bool flip; + + @override + void paint(Canvas canvas, Size size) { + if (flip) { + canvas.scale(-1, 1); + canvas.drawImage(image, Offset(-size.width, 0), + Paint()..filterQuality = ui.FilterQuality.high); + } else { + canvas.drawImage( + image, Offset(0, 0), Paint()..filterQuality = ui.FilterQuality.high); + } + } + + @override + bool shouldRepaint(CustomPainter oldDelegate) { + return false; + } +} pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy