diff --git a/frontend/app/player/web/assist/CanvasReceiver.ts b/frontend/app/player/web/assist/CanvasReceiver.ts index 325860c37..8ff54a645 100644 --- a/frontend/app/player/web/assist/CanvasReceiver.ts +++ b/frontend/app/player/web/assist/CanvasReceiver.ts @@ -33,10 +33,10 @@ export default class CanvasReceiver { path: '/assist', port: urlObject.port === '' - ? location.protocol === 'https:' - ? 443 - : 80 - : parseInt(urlObject.port), + ? location.protocol === 'https:' + ? 443 + : 80 + : parseInt(urlObject.port), }; if (this.config) { peerOpts['config'] = { @@ -68,7 +68,7 @@ export default class CanvasReceiver { (node.node as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D ); } - }, 500); + }, 250); }); call.on('error', (err) => console.error('canvas call error', err)); }); @@ -93,7 +93,28 @@ function spawnVideo(stream: MediaStream, node: VElement) { videoEl.setAttribute('muted', 'true'); videoEl.setAttribute('playsinline', 'true'); videoEl.setAttribute('crossorigin', 'anonymous'); - void videoEl.play(); + + videoEl.play() + .then(() => true) + .catch(() => { + // we allow that if user just reloaded the page + }) + + const clearListeners = () => { + document.removeEventListener('click', startStream) + videoEl.removeEventListener('playing', clearListeners) + } + videoEl.addEventListener('playing', clearListeners) + + const startStream = () => { + videoEl.play() + .then(() => console.log('unpaused')) + .catch(() => { + // we allow that if user just reloaded the page + }) + document.removeEventListener('click', startStream) + } + document.addEventListener('click', startStream) return videoEl; } diff --git a/tracker/tracker-assist/CHANGELOG.md b/tracker/tracker-assist/CHANGELOG.md index 11f9edc3e..7ab25ece5 100644 --- a/tracker/tracker-assist/CHANGELOG.md +++ b/tracker/tracker-assist/CHANGELOG.md @@ -1,3 +1,7 @@ +## 8.0.4 + +- fix for multiple canvas streaming + ## 8.0.3 - make assist canvas layer hidden from canvas capturer diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index db8d037ed..131a03de3 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "8.0.3", + "version": "8.0.4", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 307235682..aa6f3543e 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -5,7 +5,7 @@ import Peer, { MediaConnection, } from 'peerjs' import type { Properties, } from 'csstype' import { App, } from '@openreplay/tracker' -import RequestLocalStream, { LocalStream, } from './LocalStream.js' +import RequestLocalStream, { LocalStream } from './LocalStream.js'; import {hasTag,} from './guards.js' import RemoteControl, { RCStatus, } from './RemoteControl.js' import CallWindow from './CallWindow.js' @@ -79,7 +79,7 @@ export default class Assist { private socket: Socket | null = null private peer: Peer | null = null - private canvasPeer: Peer | null = null + private canvasPeers: Record = {} private assistDemandedRestart = false private callingState: CallingState = CallingState.False private remoteControl: RemoteControl | null = null; @@ -590,32 +590,41 @@ export default class Assist { }) }) + + const startCanvasStream = (stream: MediaStream, id: number) => { + const canvasPID = `${app.getProjectKey()}-${sessionId}-${id}` + if (!this.canvasPeers[id]) { + this.canvasPeers[id] = new safeCastedPeer(canvasPID, peerOptions) as Peer + } + this.canvasPeers[id]?.on('error', (e) => app.debug.error(e)) + + Object.values(this.agents).forEach(agent => { + if (agent.agentInfo) { + const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas` + const connection = this.canvasPeers[id]?.connect(target) + connection?.on('open', () => { + if (agent.agentInfo) { + const call = this.canvasPeers[id]?.call(target, stream.clone()) + call?.on('error', app.debug.error) + } + }) + connection?.on('error', (e) => app.debug.error(e)) + } else { + app.debug.error('Assist: cant establish canvas peer to agent, no agent info') + } + }) + } + app.nodes.attachNodeCallback((node) => { const id = app.nodes.getID(node) if (id && hasTag(node, 'canvas')) { - const canvasPId = `${app.getProjectKey()}-${sessionId}-${id}` - if (!this.canvasPeer) this.canvasPeer = new safeCastedPeer(canvasPId, peerOptions) as Peer + app.debug.log(`Creating stream for canvas ${id}`) const canvasHandler = new Canvas( node as unknown as HTMLCanvasElement, id, 30, (stream: MediaStream) => { - Object.values(this.agents).forEach(agent => { - if (agent.agentInfo) { - const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas` - const connection = this.canvasPeer?.connect(target) - connection?.on('open', () => { - if (agent.agentInfo) { - const pCall = this.canvasPeer?.call(target, stream) - pCall?.on('error', app.debug.error) - } - }) - connection?.on('error', app.debug.error) - this.canvasPeer?.on('error', app.debug.error) - } else { - app.debug.error('Assist: cant establish canvas peer to agent, no agent info') - } - }) + startCanvasStream(stream, id) }, app.debug.error, ) @@ -627,10 +636,10 @@ export default class Assist { private playNotificationSound() { if ('Audio' in window) { new Audio('https://static.openreplay.com/tracker-assist/notification.mp3') - .play() - .catch(e => { - this.app.debug.warn(e) - }) + .play() + .catch(e => { + this.app.debug.warn(e) + }) } } diff --git a/tracker/tracker-assist/src/version.ts b/tracker/tracker-assist/src/version.ts index 5a09f669a..65fd19713 100644 --- a/tracker/tracker-assist/src/version.ts +++ b/tracker/tracker-assist/src/version.ts @@ -1 +1 @@ -export const pkgVersion = '8.0.2' +export const pkgVersion = "8.0.4"; diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index a2872a877..bbc27aaa9 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -123,6 +123,7 @@ type AppOptions = { forceSingleTab?: boolean disableStringDict?: boolean assistSocketHost?: string + disableCanvas?: boolean /** @deprecated */ onStart?: StartCallback @@ -213,6 +214,7 @@ export default class App { forceSingleTab: false, assistSocketHost: '', fixedCanvasScaling: false, + disableCanvas: false, }, options, ) @@ -1058,7 +1060,7 @@ export default class App { await this.tagWatcher.fetchTags(this.options.ingestPoint, token) this.activityState = ActivityState.Active - if (canvasEnabled) { + if (canvasEnabled && !this.options.disableCanvas) { this.canvasRecorder = this.canvasRecorder ?? new CanvasRecorder(this, {