From c39ed684547f82905d2b0d077145a45740f5c676 Mon Sep 17 00:00:00 2001 From: sylenien Date: Fri, 14 Oct 2022 15:53:43 +0200 Subject: [PATCH] fix(player/tracker): fix camera and socket connection --- .../Assist/ChatWindow/ChatWindow.tsx | 70 +++++++++++++------ .../managers/AssistManager.ts | 18 +++-- frontend/app/player/singletone.js | 2 + tracker/tracker-assist/src/Assist.ts | 8 +-- tracker/tracker-assist/src/CallWindow.ts | 13 ++-- 5 files changed, 74 insertions(+), 37 deletions(-) diff --git a/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx b/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx index 2e51157de..35225c5d5 100644 --- a/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx +++ b/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx @@ -1,29 +1,35 @@ -import React, { useState, useEffect } from 'react' -import VideoContainer from '../components/VideoContainer' -import cn from 'classnames' -import Counter from 'App/components/shared/SessionItem/Counter' -import stl from './chatWindow.module.css' -import ChatControls from '../ChatControls/ChatControls' +import React, { useState, useEffect } from 'react'; +import VideoContainer from '../components/VideoContainer'; +import cn from 'classnames'; +import Counter from 'App/components/shared/SessionItem/Counter'; +import stl from './chatWindow.module.css'; +import ChatControls from '../ChatControls/ChatControls'; import Draggable from 'react-draggable'; import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream'; +import { toggleVideoLocalStream } from 'Player' export interface Props { - incomeStream: MediaStream[] | null, - localStream: LocalStream | null, - userId: string, + incomeStream: MediaStream[] | null; + localStream: LocalStream | null; + userId: string; isPrestart?: boolean; - endCall: () => void + endCall: () => void; } function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }: Props) { - const [localVideoEnabled, setLocalVideoEnabled] = useState(false) - const [anyRemoteEnabled, setRemoteEnabled] = useState(false) + const [localVideoEnabled, setLocalVideoEnabled] = useState(false); + const [anyRemoteEnabled, setRemoteEnabled] = useState(false); + + const onlyLocalEnabled = localVideoEnabled && !anyRemoteEnabled; + + useEffect(() => { + toggleVideoLocalStream(localVideoEnabled) + }, [localVideoEnabled]) - const onlyLocalEnabled = localVideoEnabled && !anyRemoteEnabled return (
@@ -32,21 +38,39 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
{incomeStream && incomeStream.length > 2 ? ' (+ other agents in the call)' : ''}
- +
-
- {incomeStream - ? incomeStream.map(stream => ) : ( +
+ {incomeStream ? ( + incomeStream.map((stream) => ( + + + + )) + ) : (
Error obtaining incoming streams
)} -
- +
+
- +
- ) + ); } -export default ChatWindow +export default ChatWindow; diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index 06e8adb30..83b89880a 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -205,9 +205,12 @@ export default class AssistManager { } }) socket.on('videofeed', ({ streamId, enabled }) => { + console.log(streamId, enabled) + console.log(this.videoStreams) if (this.videoStreams[streamId]) { this.videoStreams[streamId].enabled = enabled } + console.log(this.videoStreams) }) socket.on('SESSION_DISCONNECTED', e => { waitingForMessages = true @@ -370,10 +373,8 @@ export default class AssistManager { }) call.on('stream', stream => { + this.videoStreams[call.peer] = stream.getVideoTracks()[0] this.callArgs && this.callArgs.onStream(stream) - stream.getVideoTracks().forEach(track => { - this.videoStreams[track.id] = track - }) }); // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) @@ -505,9 +506,8 @@ export default class AssistManager { call.on('stream', stream => { getState().calling !== CallingState.OnCall && update({ calling: CallingState.OnCall }) - stream.getVideoTracks().forEach(track => { - this.videoStreams[track.id] = track - }) + this.videoStreams[call.peer] = stream.getVideoTracks()[0] + this.callArgs && this.callArgs.onStream(stream) }); // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) @@ -562,6 +562,12 @@ export default class AssistManager { } } + toggleVideoLocalStream(enabled: boolean) { + this.getPeer().then((peer) => { + this.socket.emit('videofeed', { streamId: peer.id, enabled }) + }) + } + private annot: AnnotationCanvas | null = null /* ==== Cleaning ==== */ diff --git a/frontend/app/player/singletone.js b/frontend/app/player/singletone.js index 56f1d59f6..1c1a856e6 100644 --- a/frontend/app/player/singletone.js +++ b/frontend/app/player/singletone.js @@ -86,6 +86,8 @@ export const jumpToLive = initCheck((...args) => instance.jumpToLive(...args)) export const toggleUserName = initCheck((...args) => instance.toggleUserName(...args)) export const injectNotes = initCheck((...args) => instance.injectNotes(...args)) export const filterOutNote = initCheck((...args) => instance.filterOutNote(...args)) +/** @type {Player.assistManager.toggleVideoLocalStream} */ +export const toggleVideoLocalStream = initCheck((...args) => instance.assistManager.toggleVideoLocalStream(...args)) export const Controls = { jump, diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index f794f59d3..ea9281455 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -248,8 +248,8 @@ export default class Assist { callingAgents.set(id, name) updateCallerNames() }) - socket.on('videofeed', ({ streamId, enabled, }) => { - callUI?.toggleVideoStream({ streamId, enabled, }) + socket.on('videofeed', (id, feedState) => { + callUI?.toggleVideoStream(feedState) }) const callingAgents: Map = new Map() // !! uses socket.io ID @@ -341,7 +341,7 @@ export default class Assist { this.emit('call_end') handleCallEnd() } - const updateVideoFeed = ({ streamId, enabled, }) => this.emit('videofeed', { streamId, enabled, }) + const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, }) peer.on('call', (call) => { app.debug.log('Incoming call: ', call) @@ -399,7 +399,7 @@ export default class Assist { initiateCallEnd() }) call.on('stream', (rStream) => { - callUI?.addRemoteStream(rStream) + callUI?.addRemoteStream(rStream, call.peer) const onInteraction = () => { // do only if document.hidden ? callUI?.playRemote() document.removeEventListener('click', onInteraction) diff --git a/tracker/tracker-assist/src/CallWindow.ts b/tracker/tracker-assist/src/CallWindow.ts index 9adc7a5ae..6b64fae3e 100644 --- a/tracker/tracker-assist/src/CallWindow.ts +++ b/tracker/tracker-assist/src/CallWindow.ts @@ -4,6 +4,7 @@ import attachDND from './dnd.js' const SS_START_TS_KEY = '__openreplay_assist_call_start_ts' export default class CallWindow { + private remoteVideoId: string private readonly iframe: HTMLIFrameElement private vRemote: HTMLVideoElement | null = null private vLocal: HTMLVideoElement | null = null @@ -30,7 +31,7 @@ export default class CallWindow { position: 'fixed', zIndex: 2147483647 - 1, border: 'none', - bottom: '10px', + bottom: '50px', right: '10px', height: '200px', width: '200px', @@ -127,13 +128,14 @@ export default class CallWindow { private checkRemoteVideoInterval: ReturnType private audioContainer: HTMLDivElement | null = null - addRemoteStream(rStream: MediaStream) { + addRemoteStream(rStream: MediaStream, peerId: string) { this.load .then(() => { // Video if (this.vRemote && !this.vRemote.srcObject) { this.vRemote.srcObject = rStream this.remoteVideo = rStream.getVideoTracks()[0] + this.remoteVideoId = peerId if (this.vPlaceholder) { this.vPlaceholder.innerText = 'Video has been paused. Click anywhere to resume.' @@ -177,6 +179,8 @@ export default class CallWindow { this.remoteVideoOn = enable if (enable) { this.videoContainer.classList.add('remote') + } else { + this.videoContainer.classList.remove('remote') } this.adjustIframeSize() } @@ -320,9 +324,10 @@ export default class CallWindow { this.localStreams = [] } - toggleVideoStream({ streamId, enabled, }: { streamId: string, enabled: boolean}) { - if (this.remoteVideo.id === streamId) { + toggleVideoStream({ streamId, enabled, }: { streamId: string, enabled: boolean }) { + if (this.remoteVideoId === streamId) { this.remoteVideo.enabled = enabled + this.toggleRemoteVideoUI(enabled) } } }