diff --git a/.github/workflows/tracker-tests.yaml b/.github/workflows/tracker-tests.yaml index d5c4caeaf..fa73f34a6 100644 --- a/.github/workflows/tracker-tests.yaml +++ b/.github/workflows/tracker-tests.yaml @@ -47,10 +47,6 @@ jobs: run: | cd tracker/tracker bun install - - name: (TA) Setup Testing packages - run: | - cd tracker/tracker-assist - bun install - name: Jest tests run: | cd tracker/tracker @@ -59,6 +55,10 @@ jobs: run: | cd tracker/tracker bun run build + - name: (TA) Setup Testing packages + run: | + cd tracker/tracker-assist + bun install - name: (TA) Jest tests run: | cd tracker/tracker-assist diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index fca1a2065..bd7716f7b 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -10,5 +10,5 @@ func IsIOSType(id int) bool { } func IsDOMType(id int) bool { - return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 117 == id || 118 == id || 93 == id || 96 == id || 100 == id || 101 == id || 102 == id || 103 == id || 104 == id || 105 == id || 106 == id || 111 == id + return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 117 == id || 118 == id || 119 == id || 93 == id || 96 == id || 100 == id || 101 == id || 102 == id || 103 == id || 104 == id || 105 == id || 106 == id || 111 == id } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 720670c71..3b9f4ef0d 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -84,6 +84,7 @@ const ( MsgResourceTiming = 116 MsgTabChange = 117 MsgTabData = 118 + MsgCanvasNode = 119 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 @@ -2245,6 +2246,29 @@ func (msg *TabData) TypeID() int { return 118 } +type CanvasNode struct { + message + NodeId string + Timestamp uint64 +} + +func (msg *CanvasNode) Encode() []byte { + buf := make([]byte, 21+len(msg.NodeId)) + buf[0] = 119 + p := 1 + p = WriteString(msg.NodeId, buf, p) + p = WriteUint(msg.Timestamp, buf, p) + return buf[:p] +} + +func (msg *CanvasNode) Decode() Message { + return msg +} + +func (msg *CanvasNode) TypeID() int { + return 119 +} + type IssueEvent struct { message MessageID uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 3260c0fed..910922b8a 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1365,6 +1365,18 @@ func DecodeTabData(reader BytesReader) (Message, error) { return msg, err } +func DecodeCanvasNode(reader BytesReader) (Message, error) { + var err error = nil + msg := &CanvasNode{} + if msg.NodeId, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1993,6 +2005,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeTabChange(reader) case 118: return DecodeTabData(reader) + case 119: + return DecodeCanvasNode(reader) case 125: return DecodeIssueEvent(reader) case 126: diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..3d0feafea --- /dev/null +++ b/codecov.yml @@ -0,0 +1,10 @@ +ignore: + - "**/*/*.gen.ts" + - "**/*/coverage.xml" + - "**/*/coverage-final.json" + - "**/*/coverage/**" + - "**/*/node_modules/**" + - "**/*/dist/**" + - "**/*/build/**" + - "**/*/.test.*" + - "**/*/version.ts" \ No newline at end of file diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 9e3091142..d5348a9e5 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -788,6 +788,14 @@ class TabData(Message): self.tab_id = tab_id +class CanvasNode(Message): + __id__ = 119 + + def __init__(self, node_id, timestamp): + self.node_id = node_id + self.timestamp = timestamp + + class IssueEvent(Message): __id__ = 125 diff --git a/ee/connectors/msgcodec/messages.pyx b/ee/connectors/msgcodec/messages.pyx index 8c1b6206a..af0816b95 100644 --- a/ee/connectors/msgcodec/messages.pyx +++ b/ee/connectors/msgcodec/messages.pyx @@ -1164,6 +1164,17 @@ cdef class TabData(PyMessage): self.tab_id = tab_id +cdef class CanvasNode(PyMessage): + cdef public int __id__ + cdef public str node_id + cdef public unsigned long timestamp + + def __init__(self, str node_id, unsigned long timestamp): + self.__id__ = 119 + self.node_id = node_id + self.timestamp = timestamp + + cdef class IssueEvent(PyMessage): cdef public int __id__ cdef public unsigned long message_id diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 0040064c8..4aba0f775 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -711,6 +711,12 @@ class MessageCodec(Codec): tab_id=self.read_string(reader) ) + if message_id == 119: + return CanvasNode( + node_id=self.read_string(reader), + timestamp=self.read_uint(reader) + ) + if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), diff --git a/ee/connectors/msgcodec/msgcodec.pyx b/ee/connectors/msgcodec/msgcodec.pyx index 6a6faf871..c918be6ea 100644 --- a/ee/connectors/msgcodec/msgcodec.pyx +++ b/ee/connectors/msgcodec/msgcodec.pyx @@ -809,6 +809,12 @@ cdef class MessageCodec: tab_id=self.read_string(reader) ) + if message_id == 119: + return CanvasNode( + node_id=self.read_string(reader), + timestamp=self.read_uint(reader) + ) + if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), diff --git a/frontend/.babelrc b/frontend/.babelrc index 631979df1..9816ce275 100644 --- a/frontend/.babelrc +++ b/frontend/.babelrc @@ -6,10 +6,10 @@ ], "plugins": [ "babel-plugin-react-require", - [ "@babel/plugin-proposal-private-property-in-object", { "loose": true } ], + ["@babel/plugin-transform-private-property-in-object", { "loose":true } ], [ "@babel/plugin-transform-runtime", { "regenerator": true } ], [ "@babel/plugin-proposal-decorators", { "legacy":true } ], - [ "@babel/plugin-proposal-class-properties", { "loose":true } ], - [ "@babel/plugin-proposal-private-methods", { "loose": true }] + [ "@babel/plugin-transform-class-properties", { "loose":true } ], + [ "@babel/plugin-transform-private-methods", { "loose": true }] ] } \ No newline at end of file diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index dc1441e69..f7ea33245 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -236,6 +236,10 @@ export default class MessageManager { } } + public getNode(id: number) { + return this.tabs[this.activeTab]?.getNode(id); + } + public changeTab(tabId: string) { this.activeTab = tabId; this.state.update({ currentTab: tabId }); diff --git a/frontend/app/player/web/Screen/Screen.ts b/frontend/app/player/web/Screen/Screen.ts index 2956940e3..1db5da88c 100644 --- a/frontend/app/player/web/Screen/Screen.ts +++ b/frontend/app/player/web/Screen/Screen.ts @@ -120,7 +120,7 @@ export default class Screen { return this.parentElement } - setBorderStyle(style: { border: string }) { + setBorderStyle(style: { outline: string }) { return Object.assign(this.screen.style, style) } diff --git a/frontend/app/player/web/TabManager.ts b/frontend/app/player/web/TabManager.ts index defde829b..2dc7c3554 100644 --- a/frontend/app/player/web/TabManager.ts +++ b/frontend/app/player/web/TabManager.ts @@ -1,23 +1,28 @@ +import type { Store } from "Player"; +import { getResourceFromNetworkRequest, getResourceFromResourceTiming, Log, ResourceType } from "Player"; import ListWalker from "Player/common/ListWalker"; +import Lists, { INITIAL_STATE as LISTS_INITIAL_STATE, InitialLists, State as ListsState } from "Player/web/Lists"; +import CanvasManager from "Player/web/managers/CanvasManager"; +import { VElement } from "Player/web/managers/DOM/VirtualDOM"; +import PagesManager from "Player/web/managers/PagesManager"; +import PerformanceTrackManager from "Player/web/managers/PerformanceTrackManager"; +import WindowNodeCounter from "Player/web/managers/WindowNodeCounter"; import { + CanvasNode, ConnectionInformation, - Message, MType, ResourceTiming, + Message, + MType, + ResourceTiming, SetPageLocation, SetViewportScroll, SetViewportSize } from "Player/web/messages"; -import PerformanceTrackManager from "Player/web/managers/PerformanceTrackManager"; -import WindowNodeCounter from "Player/web/managers/WindowNodeCounter"; -import PagesManager from "Player/web/managers/PagesManager"; +import { isDOMType } from "Player/web/messages/filters.gen"; +import Screen from "Player/web/Screen/Screen"; // @ts-ignore import { Decoder } from "syncod"; -import Lists, { InitialLists, INITIAL_STATE as LISTS_INITIAL_STATE, State as ListsState } from "Player/web/Lists"; -import type { Store } from 'Player'; -import Screen from "Player/web/Screen/Screen"; import { TYPES as EVENT_TYPES } from "Types/session/event"; -import type { PerformanceChartPoint } from './managers/PerformanceTrackManager'; -import { getResourceFromNetworkRequest, getResourceFromResourceTiming, Log, ResourceType } from "Player"; -import { isDOMType } from "Player/web/messages/filters.gen"; +import type { PerformanceChartPoint } from "./managers/PerformanceTrackManager"; export interface TabState extends ListsState { performanceAvailability?: PerformanceTrackManager['availability'] @@ -57,6 +62,8 @@ export default class TabSessionManager { public readonly decoder = new Decoder(); private lists: Lists; private navigationStartOffset = 0 + private canvasManagers: { [key: string]: { manager: CanvasManager, start: number, running: boolean } } = {} + private canvasReplayWalker: ListWalker = new ListWalker(); constructor( private readonly session: any, @@ -76,6 +83,10 @@ export default class TabSessionManager { }) } + public getNode = (id: number) => { + return this.pagesManager.getNode(id) + } + public updateLists(lists: Partial) { Object.keys(lists).forEach((key: 'event' | 'stack' | 'exceptions') => { const currentList = this.lists.lists[key] @@ -141,6 +152,23 @@ export default class TabSessionManager { distributeMessage(msg: Message): void { switch (msg.tp) { + case MType.CanvasNode: + const managerId = `${msg.timestamp}_${msg.nodeId}`; + if (!this.canvasManagers[managerId]) { + const filename = `${managerId}.mp4`; + const delta = msg.timestamp - this.sessionStart; + const fileUrl = this.session.canvasURL.find((url: string) => url.includes(filename)); + const manager = new CanvasManager( + msg.nodeId, + delta, + fileUrl, + this.getNode as (id: number) => VElement | undefined + ); + this.canvasManagers[managerId] = { manager, start: msg.timestamp, running: false }; + this.canvasReplayWalker.append(msg); + + } + break; case MType.SetPageLocation: this.locationManager.append(msg); if (msg.navigationStart > 0) { @@ -289,6 +317,16 @@ export default class TabSessionManager { if (!!lastScroll && this.screen.window) { this.screen.window.scrollTo(lastScroll.x, lastScroll.y); } + const canvasMsg = this.canvasReplayWalker.moveGetLast(t) + if (canvasMsg) { + this.canvasManagers[`${canvasMsg.timestamp}_${canvasMsg.nodeId}`].manager.startVideo(); + this.canvasManagers[`${canvasMsg.timestamp}_${canvasMsg.nodeId}`].running = true; + } + const runningManagers = Object.keys(this.canvasManagers).filter((key) => this.canvasManagers[key].running); + runningManagers.forEach((key) => { + const manager = this.canvasManagers[key].manager; + manager.move(t); + }) }) } diff --git a/frontend/app/player/web/WebLivePlayer.ts b/frontend/app/player/web/WebLivePlayer.ts index e7ec09d2c..a163eb83e 100644 --- a/frontend/app/player/web/WebLivePlayer.ts +++ b/frontend/app/player/web/WebLivePlayer.ts @@ -42,6 +42,7 @@ export default class WebLivePlayer extends WebPlayer { this.screen, config, wpState, + (id) => this.messageManager.getNode(id), uiErrorHandler, ) this.assistManager.connect(session.agentToken!, agentId, projectId) diff --git a/frontend/app/player/web/assist/AssistManager.ts b/frontend/app/player/web/assist/AssistManager.ts index ddae2a5ef..5fccb2e3d 100644 --- a/frontend/app/player/web/assist/AssistManager.ts +++ b/frontend/app/player/web/assist/AssistManager.ts @@ -1,19 +1,16 @@ +import MessageManager from 'Player/web/MessageManager'; import type { Socket } from 'socket.io-client'; import type Screen from '../Screen/Screen'; -import type { Store } from '../../common/types' +import type { Store } from '../../common/types'; import type { Message } from '../messages'; import MStreamReader from '../messages/MStreamReader'; -import JSONRawMessageReader from '../messages/JSONRawMessageReader' +import JSONRawMessageReader from '../messages/JSONRawMessageReader'; import Call, { CallingState } from './Call'; -import RemoteControl, { RemoteControlStatus } from './RemoteControl' -import ScreenRecording, { SessionRecordingStatus } from './ScreenRecording' +import RemoteControl, { RemoteControlStatus } from './RemoteControl'; +import ScreenRecording, { SessionRecordingStatus } from './ScreenRecording'; +import CanvasReceiver from 'Player/web/assist/CanvasReceiver'; - -export { - RemoteControlStatus, - SessionRecordingStatus, - CallingState, -} +export { RemoteControlStatus, SessionRecordingStatus, CallingState }; export enum ConnectionStatus { Connecting, @@ -25,29 +22,30 @@ export enum ConnectionStatus { Closed, } -type StatsEvent = 's_call_started' +type StatsEvent = + | 's_call_started' | 's_call_ended' | 's_control_started' | 's_control_ended' | 's_recording_started' - | 's_recording_ended' + | 's_recording_ended'; export function getStatusText(status: ConnectionStatus): string { - switch(status) { + switch (status) { case ConnectionStatus.Closed: return 'Closed...'; case ConnectionStatus.Connecting: - return "Connecting..."; + return 'Connecting...'; case ConnectionStatus.Connected: - return ""; + return ''; case ConnectionStatus.Inactive: - return "Client tab is inactive"; + return 'Client tab is inactive'; case ConnectionStatus.Disconnected: - return "Disconnected"; + return 'Disconnected'; case ConnectionStatus.Error: - return "Something went wrong. Try to reload the page."; + return 'Something went wrong. Try to reload the page.'; case ConnectionStatus.WaitingMessages: - return "Connected. Waiting for the data... (The tab might be inactive)" + return 'Connected. Waiting for the data... (The tab might be inactive)'; } } @@ -59,14 +57,16 @@ export function getStatusText(status: ConnectionStatus): string { const MAX_RECONNECTION_COUNT = 4; export default class AssistManager { - assistVersion = 1 + assistVersion = 1; + private canvasReceiver: CanvasReceiver; static readonly INITIAL_STATE = { peerConnectionStatus: ConnectionStatus.Connecting, assistStart: 0, ...Call.INITIAL_STATE, ...RemoteControl.INITIAL_STATE, ...ScreenRecording.INITIAL_STATE, - } + }; + // TODO: Session type constructor( private session: any, @@ -75,26 +75,31 @@ export default class AssistManager { private screen: Screen, private config: RTCIceServer[] | null, private store: Store, - public readonly uiErrorHandler?: { error: (msg: string) => void } + private getNode: MessageManager['getNode'], + public readonly uiErrorHandler?: { + error: (msg: string) => void; + } ) {} - public getAssistVersion = () => this.assistVersion + public getAssistVersion = () => this.assistVersion; private get borderStyle() { - const { recordingState, remoteControl } = this.store.get() + const { recordingState, remoteControl } = this.store.get(); - const isRecordingActive = recordingState === SessionRecordingStatus.Recording - const isControlActive = remoteControl === RemoteControlStatus.Enabled + const isRecordingActive = recordingState === SessionRecordingStatus.Recording; + const isControlActive = remoteControl === RemoteControlStatus.Enabled; // recording gets priority here - if (isRecordingActive) return { outline: '2px dashed red' } - if (isControlActive) return { outline: '2px dashed blue' } - return { outline: 'unset' } + if (isRecordingActive) return { outline: '2px dashed red' }; + if (isControlActive) return { outline: '2px dashed blue' }; + return { outline: 'unset' }; } private setStatus(status: ConnectionStatus) { - if (this.store.get().peerConnectionStatus === ConnectionStatus.Disconnected && - status !== ConnectionStatus.Connected) { - return + if ( + this.store.get().peerConnectionStatus === ConnectionStatus.Disconnected && + status !== ConnectionStatus.Connected + ) { + return; } if (status === ConnectionStatus.Connecting) { @@ -111,145 +116,154 @@ export default class AssistManager { } private get peerID(): string { - return `${this.session.projectKey}-${this.session.sessionId}` + return `${this.session.projectKey}-${this.session.sessionId}`; } - private socketCloseTimeout: ReturnType | undefined + private socketCloseTimeout: ReturnType | undefined; private onVisChange = () => { - this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout) + this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout); if (document.hidden) { this.socketCloseTimeout = setTimeout(() => { - const state = this.store.get() - if (document.hidden && + const state = this.store.get(); + if ( + document.hidden && // TODO: should it be RemoteControlStatus.Disabled? (check) - (state.calling === CallingState.NoCall && state.remoteControl === RemoteControlStatus.Enabled)) { - this.socket?.close() + state.calling === CallingState.NoCall && + state.remoteControl === RemoteControlStatus.Enabled + ) { + this.socket?.close(); } - }, 30000) + }, 30000); } else { - this.socket?.open() + this.socket?.open(); } - } + }; + + private socket: Socket | null = null; + private disconnectTimeout: ReturnType | undefined; + private inactiveTimeout: ReturnType | undefined; + private inactiveTabs: string[] = []; - private socket: Socket | null = null - private disconnectTimeout: ReturnType | undefined - private inactiveTimeout: ReturnType | undefined - private inactiveTabs: string[] = [] private clearDisconnectTimeout() { - this.disconnectTimeout && clearTimeout(this.disconnectTimeout) - this.disconnectTimeout = undefined + this.disconnectTimeout && clearTimeout(this.disconnectTimeout); + this.disconnectTimeout = undefined; } - private clearInactiveTimeout() { - this.inactiveTimeout && clearTimeout(this.inactiveTimeout) - this.inactiveTimeout = undefined - } - connect(agentToken: string, agentId: number, projectId: number) { - const jmr = new JSONRawMessageReader() - const reader = new MStreamReader(jmr, this.session.startedAt) - let waitingForMessages = true - const now = +new Date() - this.store.update({ assistStart: now }) + private clearInactiveTimeout() { + this.inactiveTimeout && clearTimeout(this.inactiveTimeout); + this.inactiveTimeout = undefined; + } + + connect(agentToken: string, agentId: number, projectId: number) { + const jmr = new JSONRawMessageReader(); + const reader = new MStreamReader(jmr, this.session.startedAt); + let waitingForMessages = true; + + const now = +new Date(); + this.store.update({ assistStart: now }); // @ts-ignore import('socket.io-client').then(({ default: io }) => { - if (this.socket != null || this.cleaned) { return } + if (this.socket != null || this.cleaned) { + return; + } // @ts-ignore - const urlObject = new URL(window.env.API_EDP || window.location.origin) // does it handle ssl automatically? + const urlObject = new URL(window.env.API_EDP || window.location.origin); // does it handle ssl automatically? - const socket: Socket = this.socket = io(urlObject.origin, { + const socket: Socket = (this.socket = io(urlObject.origin, { withCredentials: true, multiplex: true, transports: ['websocket'], path: '/ws-assist/socket', auth: { - token: agentToken + token: agentToken, }, query: { peerId: this.peerID, projectId, - identity: "agent", + identity: 'agent', agentInfo: JSON.stringify({ ...this.session.agentInfo, id: agentId, + peerId: this.peerID, query: document.location.search, - }) - } - }) - socket.on("connect", () => { - waitingForMessages = true - this.setStatus(ConnectionStatus.WaitingMessages) // TODO: reconnect happens frequently on bad network - }) + }), + }, + })); + socket.on('connect', () => { + waitingForMessages = true; + this.setStatus(ConnectionStatus.WaitingMessages); // TODO: reconnect happens frequently on bad network + }); - socket.on('messages', messages => { - const isOldVersion = messages.meta.version === 1 - this.assistVersion = isOldVersion ? 1 : 2 + socket.on('messages', (messages) => { + const isOldVersion = messages.meta.version === 1; + this.assistVersion = isOldVersion ? 1 : 2; - const data = messages.data || messages - jmr.append(data) // as RawMessage[] + const data = messages.data || messages; + jmr.append(data); // as RawMessage[] if (waitingForMessages) { - waitingForMessages = false // TODO: more explicit - this.setStatus(ConnectionStatus.Connected) + waitingForMessages = false; // TODO: more explicit + this.setStatus(ConnectionStatus.Connected); } if (messages.meta.tabId !== this.store.get().currentTab) { - this.clearDisconnectTimeout() + this.clearDisconnectTimeout(); if (isOldVersion) { - reader.currentTab = messages.meta.tabId - this.store.update({ currentTab: messages.meta.tabId }) + reader.currentTab = messages.meta.tabId; + this.store.update({ currentTab: messages.meta.tabId }); } } - for (let msg = reader.readNext();msg !== null;msg = reader.readNext()) { - this.handleMessage(msg, msg._index) + for (let msg = reader.readNext(); msg !== null; msg = reader.readNext()) { + this.handleMessage(msg, msg._index); } - }) + }); socket.on('SESSION_RECONNECTED', () => { - this.clearDisconnectTimeout() - this.clearInactiveTimeout() - this.setStatus(ConnectionStatus.Connected) - }) + this.clearDisconnectTimeout(); + this.clearInactiveTimeout(); + this.setStatus(ConnectionStatus.Connected); + }); socket.on('UPDATE_SESSION', (evData) => { - const { meta = {}, data = {} } = evData - const { tabId } = meta - const usedData = this.assistVersion === 1 ? evData : data - const { active } = usedData - const currentTab = this.store.get().currentTab - this.clearDisconnectTimeout() - !this.inactiveTimeout && this.setStatus(ConnectionStatus.Connected) - if (typeof active === "boolean") { - this.clearInactiveTimeout() + const { meta = {}, data = {} } = evData; + const { tabId } = meta; + const usedData = this.assistVersion === 1 ? evData : data; + const { active } = usedData; + const currentTab = this.store.get().currentTab; + this.clearDisconnectTimeout(); + !this.inactiveTimeout && this.setStatus(ConnectionStatus.Connected); + if (typeof active === 'boolean') { + this.clearInactiveTimeout(); if (active) { - this.setStatus(ConnectionStatus.Connected) - this.inactiveTabs = this.inactiveTabs.filter(t => t !== tabId) + this.setStatus(ConnectionStatus.Connected); + this.inactiveTabs = this.inactiveTabs.filter((t) => t !== tabId); } else { if (!this.inactiveTabs.includes(tabId)) { - this.inactiveTabs.push(tabId) + this.inactiveTabs.push(tabId); } if (tabId === undefined || tabId === currentTab) { this.inactiveTimeout = setTimeout(() => { // @ts-ignore - const tabs = this.store.get().tabs + const tabs = this.store.get().tabs; if (this.inactiveTabs.length === tabs.size) { - this.setStatus(ConnectionStatus.Inactive) + this.setStatus(ConnectionStatus.Inactive); } - }, 10000) + }, 10000); } } } - }) - socket.on('SESSION_DISCONNECTED', e => { - waitingForMessages = true - this.clearDisconnectTimeout() + }); + socket.on('SESSION_DISCONNECTED', (e) => { + waitingForMessages = true; + this.clearDisconnectTimeout(); this.disconnectTimeout = setTimeout(() => { - this.setStatus(ConnectionStatus.Disconnected) - }, 30000) - }) - socket.on('error', e => { - console.warn("Socket error: ", e ) + this.setStatus(ConnectionStatus.Disconnected); + }, 30000); + }); + socket.on('error', (e) => { + console.warn('Socket error: ', e); this.setStatus(ConnectionStatus.Error); - }) + }); // Maybe do lazy initialization for all? // TODO: socket proxy (depend on interfaces) @@ -259,88 +273,93 @@ export default class AssistManager { this.config, this.peerID, this.getAssistVersion - ) + ); this.remoteControl = new RemoteControl( this.store, socket, this.screen, this.session.agentInfo, () => this.screen.setBorderStyle(this.borderStyle), - this.getAssistVersion, - ) + this.getAssistVersion + ); this.screenRecording = new ScreenRecording( this.store, socket, this.session.agentInfo, () => this.screen.setBorderStyle(this.borderStyle), this.uiErrorHandler, - this.getAssistVersion, - ) + this.getAssistVersion + ); + this.canvasReceiver = new CanvasReceiver(this.peerID, this.config, this.getNode, { + ...this.session.agentInfo, + id: agentId, + }); - document.addEventListener('visibilitychange', this.onVisChange) - }) + document.addEventListener('visibilitychange', this.onVisChange); + }); } public ping(event: StatsEvent, id: number) { - this.socket?.emit(event, id) + this.socket?.emit(event, id); } - /* ==== ScreenRecording ==== */ - private screenRecording: ScreenRecording | null = null + private screenRecording: ScreenRecording | null = null; requestRecording = (...args: Parameters) => { - return this.screenRecording?.requestRecording(...args) - } + return this.screenRecording?.requestRecording(...args); + }; stopRecording = (...args: Parameters) => { - return this.screenRecording?.stopRecording(...args) - } - + return this.screenRecording?.stopRecording(...args); + }; /* ==== RemoteControl ==== */ - private remoteControl: RemoteControl | null = null - requestReleaseRemoteControl = (...args: Parameters) => { - return this.remoteControl?.requestReleaseRemoteControl(...args) - } + private remoteControl: RemoteControl | null = null; + requestReleaseRemoteControl = ( + ...args: Parameters + ) => { + return this.remoteControl?.requestReleaseRemoteControl(...args); + }; setRemoteControlCallbacks = (...args: Parameters) => { - return this.remoteControl?.setCallbacks(...args) - } + return this.remoteControl?.setCallbacks(...args); + }; releaseRemoteControl = (...args: Parameters) => { - return this.remoteControl?.releaseRemoteControl(...args) - } + return this.remoteControl?.releaseRemoteControl(...args); + }; toggleAnnotation = (...args: Parameters) => { - return this.remoteControl?.toggleAnnotation(...args) - } + return this.remoteControl?.toggleAnnotation(...args); + }; /* ==== Call ==== */ - private callManager: Call | null = null + private callManager: Call | null = null; initiateCallEnd = async (...args: Parameters) => { - return this.callManager?.initiateCallEnd(...args) - } + return this.callManager?.initiateCallEnd(...args); + }; setCallArgs = (...args: Parameters) => { - return this.callManager?.setCallArgs(...args) - } + return this.callManager?.setCallArgs(...args); + }; call = (...args: Parameters) => { - return this.callManager?.call(...args) - } + return this.callManager?.call(...args); + }; toggleVideoLocalStream = (...args: Parameters) => { - return this.callManager?.toggleVideoLocalStream(...args) - } + return this.callManager?.toggleVideoLocalStream(...args); + }; addPeerCall = (...args: Parameters) => { - return this.callManager?.addPeerCall(...args) - } - + return this.callManager?.addPeerCall(...args); + }; /* ==== Cleaning ==== */ - private cleaned = false + private cleaned = false; + clean() { - this.cleaned = true // sometimes cleaned before modules loaded - this.remoteControl?.clean() - this.callManager?.clean() - this.socket?.close() - this.socket = null - this.clearDisconnectTimeout() - this.clearInactiveTimeout() - this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout) - document.removeEventListener('visibilitychange', this.onVisChange) + this.cleaned = true; // sometimes cleaned before modules loaded + this.remoteControl?.clean(); + this.callManager?.clean(); + this.canvasReceiver?.clear(); + this.socket?.close(); + this.socket = null; + this.clearDisconnectTimeout(); + this.clearInactiveTimeout(); + this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout); + document.removeEventListener('visibilitychange', this.onVisChange); } } diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index 03f4c0134..1ad66dc15 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -32,7 +32,7 @@ export default class Call { private videoStreams: Record = {}; constructor( - private store: Store, + private store: Store }>, private socket: Socket, private config: RTCIceServer[] | null, private peerID: string, @@ -253,7 +253,7 @@ export default class Call { this.getAssistVersion() === 1 ? this.peerID : `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`; - console.log(peerId, this.getAssistVersion()); + void this._peerConnection(peerId); this.emitData('_agent_name', appStore.getState().getIn(['user', 'account', 'name'])); } diff --git a/frontend/app/player/web/assist/CanvasReceiver.ts b/frontend/app/player/web/assist/CanvasReceiver.ts new file mode 100644 index 000000000..325860c37 --- /dev/null +++ b/frontend/app/player/web/assist/CanvasReceiver.ts @@ -0,0 +1,162 @@ +import Peer from 'peerjs'; +import { VElement } from 'Player/web/managers/DOM/VirtualDOM'; +import MessageManager from 'Player/web/MessageManager'; + +let frameCounter = 0; + +function draw( + video: HTMLVideoElement, + canvas: HTMLCanvasElement, + canvasCtx: CanvasRenderingContext2D +) { + if (frameCounter % 4 === 0) { + canvasCtx.drawImage(video, 0, 0, canvas.width, canvas.height); + } + frameCounter++; + requestAnimationFrame(() => draw(video, canvas, canvasCtx)); +} + +export default class CanvasReceiver { + private streams: Map = new Map(); + private peer: Peer | null = null; + + constructor( + private readonly peerIdPrefix: string, + private readonly config: RTCIceServer[] | null, + private readonly getNode: MessageManager['getNode'], + private readonly agentInfo: Record + ) { + // @ts-ignore + const urlObject = new URL(window.env.API_EDP || window.location.origin); + const peerOpts: Peer.PeerJSOption = { + host: urlObject.hostname, + path: '/assist', + port: + urlObject.port === '' + ? location.protocol === 'https:' + ? 443 + : 80 + : parseInt(urlObject.port), + }; + if (this.config) { + peerOpts['config'] = { + iceServers: this.config, + //@ts-ignore + sdpSemantics: 'unified-plan', + iceTransportPolicy: 'all', + }; + } + const id = `${this.peerIdPrefix}-${this.agentInfo.id}-canvas`; + const canvasPeer = new Peer(id, peerOpts); + this.peer = canvasPeer; + canvasPeer.on('error', (err) => console.error('canvas peer error', err)); + canvasPeer.on('call', (call) => { + call.answer(); + const canvasId = call.peer.split('-')[2]; + call.on('stream', (stream) => { + this.streams.set(canvasId, stream); + setTimeout(() => { + const node = this.getNode(parseInt(canvasId, 10)); + const videoEl = spawnVideo( + this.streams.get(canvasId)?.clone() as MediaStream, + node as VElement + ); + if (node) { + draw( + videoEl, + node.node as HTMLCanvasElement, + (node.node as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D + ); + } + }, 500); + }); + call.on('error', (err) => console.error('canvas call error', err)); + }); + } + + clear() { + if (this.peer) { + // otherwise it calls reconnection on data chan close + const peer = this.peer; + this.peer = null; + peer.disconnect(); + peer.destroy(); + } + } +} + +function spawnVideo(stream: MediaStream, node: VElement) { + const videoEl = document.createElement('video'); + + videoEl.srcObject = stream + videoEl.setAttribute('autoplay', 'true'); + videoEl.setAttribute('muted', 'true'); + videoEl.setAttribute('playsinline', 'true'); + videoEl.setAttribute('crossorigin', 'anonymous'); + void videoEl.play(); + + return videoEl; +} + +function spawnDebugVideo(stream: MediaStream, node: VElement) { + const video = document.createElement('video'); + video.id = 'canvas-or-testing'; + video.style.border = '1px solid red'; + video.setAttribute('autoplay', 'true'); + video.setAttribute('muted', 'true'); + video.setAttribute('playsinline', 'true'); + video.setAttribute('crossorigin', 'anonymous'); + + const coords = node.node.getBoundingClientRect(); + + Object.assign(video.style, { + position: 'absolute', + left: `${coords.left}px`, + top: `${coords.top}px`, + width: `${coords.width}px`, + height: `${coords.height}px`, + }); + video.width = coords.width; + video.height = coords.height; + video.srcObject = stream; + + document.body.appendChild(video); + video + .play() + .then(() => { + console.log('started streaming canvas'); + }) + .catch((e) => { + console.error(e); + const waiter = () => { + void video.play(); + document.removeEventListener('click', waiter); + }; + document.addEventListener('click', waiter); + }); +} + +/** simple peer example + * // @ts-ignore + * const peer = new SLPeer({ initiator: false }) + * socket.on('c_signal', ({ data }) => { + * console.log('got signal', data) + * peer.signal(data.data); + * peer.canvasId = data.id; + * }); + * + * peer.on('signal', (data: any) => { + * socket.emit('c_signal', data); + * }); + * peer.on('stream', (stream: MediaStream) => { + * console.log('stream ready', stream, peer.canvasId); + * this.streams.set(peer.canvasId, stream) + * setTimeout(() => { + * const node = this.getNode(peer.canvasId) + * console.log(peer.canvasId, this.streams, node) + * spawnVideo(this.streams.get(peer.canvasId)?.clone(), node, this.screen) + * }, 500) + * }) + * peer.on('error', console.error) + * + * */ diff --git a/frontend/app/player/web/managers/CanvasManager.ts b/frontend/app/player/web/managers/CanvasManager.ts new file mode 100644 index 000000000..051e5e044 --- /dev/null +++ b/frontend/app/player/web/managers/CanvasManager.ts @@ -0,0 +1,53 @@ +import { VElement } from "Player/web/managers/DOM/VirtualDOM"; + +export default class CanvasManager { + private fileData: string | undefined; + private canvasEl: HTMLVideoElement + private canvasCtx: CanvasRenderingContext2D | null = null; + private videoTag = document.createElement('video') + private lastTs = 0; + + constructor( + private readonly nodeId: string, + private readonly delta: number, + private readonly filename: string, + private readonly getNode: (id: number) => VElement | undefined) { + // getting mp4 file composed from canvas snapshot images + fetch(this.filename).then((r) => { + if (r.status === 200) { + r.blob().then((blob) => { + this.fileData = URL.createObjectURL(blob); + }) + } else { + return Promise.reject(`File ${this.filename} not found`) + } + }).catch(console.error) + } + + startVideo = () => { + if (!this.fileData) return; + this.videoTag.setAttribute('autoplay', 'true'); + this.videoTag.setAttribute('muted', 'true'); + this.videoTag.setAttribute('playsinline', 'true'); + this.videoTag.setAttribute('crossorigin', 'anonymous'); + this.videoTag.src = this.fileData; + this.videoTag.currentTime = 0; + + const node = this.getNode(parseInt(this.nodeId, 10)) as unknown as VElement + this.canvasCtx = (node.node as HTMLCanvasElement).getContext('2d'); + this.canvasEl = node.node as HTMLVideoElement; + } + + move(t: number) { + if (t - this.lastTs < 100) return; + this.lastTs = t; + const playTime = t - this.delta + if (playTime > 0) { + if (!this.videoTag.paused) { + void this.videoTag.pause() + } + this.videoTag.currentTime = playTime/1000; + this.canvasCtx?.drawImage(this.videoTag, 0, 0, this.canvasEl.width, this.canvasEl.height); + } + } +} \ No newline at end of file diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index 18700fd50..baa30d300 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -113,6 +113,10 @@ export default class DOMManager extends ListWalker { return false; } + public getNode(id: number) { + return this.vElements.get(id) || this.vTexts.get(id) + } + private insertNode({ parentID, id, index }: { parentID: number, id: number, index: number }): void { const child = this.vElements.get(id) || this.vTexts.get(id) if (!child) { @@ -208,7 +212,8 @@ export default class DOMManager extends ListWalker { return } case MType.CreateElementNode: { - const vElem = new VElement(msg.tag, msg.svg) + // if (msg.tag.toLowerCase() === 'canvas') msg.tag = 'video' + const vElem = new VElement(msg.tag, msg.svg, msg.index) if (['STYLE', 'style', 'LINK'].includes(msg.tag)) { vElem.prioritized = true } diff --git a/frontend/app/player/web/managers/DOM/VirtualDOM.ts b/frontend/app/player/web/managers/DOM/VirtualDOM.ts index 071b89a8f..fccb5816d 100644 --- a/frontend/app/player/web/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/web/managers/DOM/VirtualDOM.ts @@ -144,7 +144,7 @@ export class VElement extends VParent { parentNode: VParent | null = null /** Should be modified only by he parent itself */ private newAttributes: Map = new Map() - constructor(readonly tagName: string, readonly isSVG = false) { super() } + constructor(readonly tagName: string, readonly isSVG = false, public readonly index: number) { super() } protected createNode() { try { return this.isSVG diff --git a/frontend/app/player/web/managers/PagesManager.ts b/frontend/app/player/web/managers/PagesManager.ts index 18722f962..4569c1648 100644 --- a/frontend/app/player/web/managers/PagesManager.ts +++ b/frontend/app/player/web/managers/PagesManager.ts @@ -56,6 +56,10 @@ export default class PagesManager extends ListWalker { this.forEach(page => page.sort(comparator)) } + public getNode(id: number) { + return this.currentPage?.getNode(id) + } + moveReady(t: number): Promise { const requiredPage = this.moveGetLast(t) if (requiredPage != null) { diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 6edd13fce..793d3967e 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -713,6 +713,16 @@ export default class RawMessageReader extends PrimitiveReader { }; } + case 119: { + const nodeId = this.readString(); if (nodeId === null) { return resetPointer() } + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + return { + tp: MType.CanvasNode, + nodeId, + timestamp, + }; + } + case 93: { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const length = this.readUint(); if (length === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/filters.gen.ts b/frontend/app/player/web/messages/filters.gen.ts index 1c8a091c6..eca3afb64 100644 --- a/frontend/app/player/web/messages/filters.gen.ts +++ b/frontend/app/player/web/messages/filters.gen.ts @@ -4,7 +4,7 @@ import { MType } from './raw.gen' const IOS_TYPES = [90,91,92,93,94,95,96,97,98,100,101,102,103,104,105,106,107,110,111] -const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,114,117,118] +const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,114,117,118,119] export function isDOMType(t: MType) { return DOM_TYPES.includes(t) } \ No newline at end of file diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index ef7922b71..efc5d6cd6 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -61,6 +61,7 @@ import type { RawResourceTiming, RawTabChange, RawTabData, + RawCanvasNode, RawIosEvent, RawIosScreenChanges, RawIosClickEvent, @@ -190,6 +191,8 @@ export type TabChange = RawTabChange & Timed export type TabData = RawTabData & Timed +export type CanvasNode = RawCanvasNode & Timed + export type IosEvent = RawIosEvent & Timed export type IosScreenChanges = RawIosScreenChanges & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index d910ee949..f808a7713 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -59,6 +59,7 @@ export const enum MType { ResourceTiming = 116, TabChange = 117, TabData = 118, + CanvasNode = 119, IosEvent = 93, IosScreenChanges = 96, IosClickEvent = 100, @@ -476,6 +477,12 @@ export interface RawTabData { tabId: string, } +export interface RawCanvasNode { + tp: MType.CanvasNode, + nodeId: string, + timestamp: number, +} + export interface RawIosEvent { tp: MType.IosEvent, timestamp: number, @@ -568,4 +575,4 @@ export interface RawIosIssueEvent { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequestDeprecated | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingDeprecated | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawNetworkRequest | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawTabChange | RawTabData | RawIosEvent | RawIosScreenChanges | RawIosClickEvent | RawIosInputEvent | RawIosPerformanceEvent | RawIosLog | RawIosInternalError | RawIosNetworkCall | RawIosSwipeEvent | RawIosIssueEvent; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequestDeprecated | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingDeprecated | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawNetworkRequest | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawTabChange | RawTabData | RawCanvasNode | RawIosEvent | RawIosScreenChanges | RawIosClickEvent | RawIosInputEvent | RawIosPerformanceEvent | RawIosLog | RawIosInternalError | RawIosNetworkCall | RawIosSwipeEvent | RawIosIssueEvent; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index 10eae3770..5ee4cf6a0 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -60,6 +60,7 @@ export const TP_MAP = { 116: MType.ResourceTiming, 117: MType.TabChange, 118: MType.TabData, + 119: MType.CanvasNode, 93: MType.IosEvent, 96: MType.IosScreenChanges, 100: MType.IosClickEvent, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index f389e4af2..d7042a355 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -493,8 +493,14 @@ type TrTabData = [ tabId: string, ] +type TrCanvasNode = [ + type: 119, + nodeId: string, + timestamp: number, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequestDeprecated | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingDeprecated | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrNetworkRequest | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming | TrTabChange | TrTabData + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequestDeprecated | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingDeprecated | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrNetworkRequest | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming | TrTabChange | TrTabData | TrCanvasNode export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -992,6 +998,14 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } + case 119: { + return { + tp: MType.CanvasNode, + nodeId: tMsg[1], + timestamp: tMsg[2], + } + } + default: return null } diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index 67eff4f8f..8700876ea 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -79,6 +79,7 @@ export interface ISession { metadata: []; favorite: boolean; filterId?: string; + canvasURL: string[]; domURL: string[]; devtoolsURL: string[]; /** @@ -148,6 +149,7 @@ const emptyValues = { devtoolsURL: [], mobsUrl: [], notes: [], + canvasURL: [], metadata: {}, startedAt: 0, platform: 'web', @@ -160,6 +162,7 @@ export default class Session { siteId: ISession['siteId']; projectKey: ISession['projectKey']; peerId: ISession['peerId']; + canvasURL: ISession['canvasURL']; live: ISession['live']; startedAt: ISession['startedAt']; duration: ISession['duration']; @@ -234,6 +237,7 @@ export default class Session { mobsUrl = [], crashes = [], notes = [], + canvasURL = [], ...session } = sessionData; const duration = Duration.fromMillis(session.duration < 1000 ? 1000 : session.duration); @@ -325,6 +329,7 @@ export default class Session { domURL, devtoolsURL, notes, + canvasURL, notesWithEvents: mixedEventsWithIssues, frustrations: frustrationList, }); diff --git a/frontend/app/window.d.ts b/frontend/app/window.d.ts new file mode 100644 index 000000000..26cd12261 --- /dev/null +++ b/frontend/app/window.d.ts @@ -0,0 +1,14 @@ +declare global { + interface Window { + env: { + NODE_ENV: string + ORIGIN: string + ASSETS_HOST: string + API_EDP: string + VERSION: string + TRACKER_VERSION: string + TRACKER_HOST: string + SOURCEMAP: boolean + } + } +} diff --git a/frontend/package.json b/frontend/package.json index e6a29f7f4..0b1c79fca 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -25,6 +25,7 @@ }, "dependencies": { "@ant-design/icons": "^5.2.5", + "@babel/plugin-transform-private-methods": "^7.23.3", "@floating-ui/react-dom-interactions": "^0.10.3", "@sentry/browser": "^5.21.1", "@svg-maps/world": "^1.0.1", @@ -89,7 +90,7 @@ "@babel/plugin-proposal-decorators": "^7.23.2", "@babel/plugin-proposal-private-methods": "^7.18.6", "@babel/plugin-syntax-bigint": "^7.8.3", - "@babel/plugin-transform-class-properties": "^7.22.5", + "@babel/plugin-transform-class-properties": "^7.23.3", "@babel/plugin-transform-private-property-in-object": "^7.22.11", "@babel/plugin-transform-runtime": "^7.17.12", "@babel/preset-env": "^7.23.2", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index b4cc35011..9290418de 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1626,6 +1626,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-class-properties@npm:^7.23.3": + version: 7.23.3 + resolution: "@babel/plugin-transform-class-properties@npm:7.23.3" + dependencies: + "@babel/helper-create-class-features-plugin": ^7.22.15 + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: bca30d576f539eef216494b56d610f1a64aa9375de4134bc021d9660f1fa735b1d7cc413029f22abc0b7cb737e3a57935c8ae9d8bd1730921ccb1deebce51bfd + languageName: node + linkType: hard + "@babel/plugin-transform-class-static-block@npm:^7.22.11": version: 7.22.11 resolution: "@babel/plugin-transform-class-static-block@npm:7.22.11" @@ -2227,6 +2239,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-private-methods@npm:^7.23.3": + version: 7.23.3 + resolution: "@babel/plugin-transform-private-methods@npm:7.23.3" + dependencies: + "@babel/helper-create-class-features-plugin": ^7.22.15 + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 745a655edcd111b7f91882b921671ca0613079760d8c9befe336b8a9bc4ce6bb49c0c08941831c950afb1b225b4b2d3eaac8842e732db095b04db38efd8c34f4 + languageName: node + linkType: hard + "@babel/plugin-transform-private-property-in-object@npm:^7.22.11": version: 7.22.11 resolution: "@babel/plugin-transform-private-property-in-object@npm:7.22.11" @@ -19630,7 +19654,8 @@ __metadata: "@babel/plugin-proposal-decorators": ^7.23.2 "@babel/plugin-proposal-private-methods": ^7.18.6 "@babel/plugin-syntax-bigint": ^7.8.3 - "@babel/plugin-transform-class-properties": ^7.22.5 + "@babel/plugin-transform-class-properties": ^7.23.3 + "@babel/plugin-transform-private-methods": ^7.23.3 "@babel/plugin-transform-private-property-in-object": ^7.22.11 "@babel/plugin-transform-runtime": ^7.17.12 "@babel/preset-env": ^7.23.2 diff --git a/mobs/messages.rb b/mobs/messages.rb index 4682538f5..538f1eeb4 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -504,6 +504,11 @@ message 118, 'TabData' do string 'TabId' end +message 119, 'CanvasNode' do + string 'NodeId' + uint 'Timestamp' +end + ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do uint 'MessageID' diff --git a/tracker/.husky/pre-commit b/tracker/.husky/pre-commit index 0200dbbae..ae22cb74b 100755 --- a/tracker/.husky/pre-commit +++ b/tracker/.husky/pre-commit @@ -1,7 +1,7 @@ #!/usr/bin/env sh . "$(dirname -- "$0")/_/husky.sh" -if git diff --cached --name-only | grep --quiet '^tracker/tracker/' +if git diff --cached --name-only | grep -v '\.gen\.ts$' | grep --quiet '^tracker/tracker/' then echo "tracker" pwd @@ -12,7 +12,7 @@ then cd ../../ fi -if git diff --cached --name-only | grep --quiet '^tracker/tracker-assist/' +if git diff --cached --name-only | grep -v '\.gen\.ts$' | grep --quiet '^tracker/tracker-assist/' then echo "tracker-assist" cd tracker/tracker-assist diff --git a/tracker/tracker-assist/bun.lockb b/tracker/tracker-assist/bun.lockb index 06287d996..2eb6b31b2 100755 Binary files a/tracker/tracker-assist/bun.lockb and b/tracker/tracker-assist/bun.lockb differ diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 1ca11de79..02ba6e018 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "6.0.3", + "version": "6.0.4-57", "keywords": [ "WebRTC", "assistance", @@ -34,7 +34,7 @@ "socket.io-client": "^4.7.2" }, "peerDependencies": { - "@openreplay/tracker": ">=8.0.0" + "@openreplay/tracker": ">=10.0.3" }, "devDependencies": { "@openreplay/tracker": "file:../tracker", diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index a52bad921..b70e5ad1e 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/no-empty-function */ +import {hasTag,} from '@openreplay/tracker/lib/app/guards' import type { Socket, } from 'socket.io-client' import { connect, } from 'socket.io-client' import Peer, { MediaConnection, } from 'peerjs' @@ -14,6 +15,7 @@ import { callConfirmDefault, } from './ConfirmWindow/defaults.js' import type { Options as ConfirmOptions, } from './ConfirmWindow/defaults.js' import ScreenRecordingState from './ScreenRecordingState.js' import { pkgVersion, } from './version.js' +import Canvas from './Canvas.js' // TODO: fully specified strict check with no-any (everywhere) // @ts-ignore @@ -21,6 +23,14 @@ const safeCastedPeer = Peer.default || Peer type StartEndCallback = (agentInfo?: Record) => ((() => any) | void) +interface AgentInfo { + email: string; + id: number + name: string + peerId: string + query: string +} + export interface Options { onAgentConnect: StartEndCallback; onCallStart: StartEndCallback; @@ -58,7 +68,7 @@ type OptionalCallback = (()=>Record) | void type Agent = { onDisconnect?: OptionalCallback, onControlReleased?: OptionalCallback, - agentInfo: Record | undefined + agentInfo: AgentInfo | undefined // } @@ -68,12 +78,15 @@ export default class Assist { private socket: Socket | null = null private peer: Peer | null = null + private canvasPeer: Peer | null = null private assistDemandedRestart = false private callingState: CallingState = CallingState.False private remoteControl: RemoteControl | null = null; private agents: Record = {} private readonly options: Options + private readonly canvasMap: Map = new Map() + constructor( private readonly app: App, options?: Partial, @@ -151,13 +164,14 @@ export default class Assist { } return '' } + private onStart() { const app = this.app const sessionId = app.getSessionID() // Common for all incoming call requests let callUI: CallWindow | null = null let annot: AnnotationCanvas | null = null - // TODO: incapsulate + // TODO: encapsulate let callConfirmWindow: ConfirmWindow | null = null let callConfirmAnswer: Promise | null = null let callEndCallback: ReturnType | null = null @@ -190,7 +204,7 @@ export default class Assist { app.debug.log('Socket:', ...args) }) - const onGrand = (id) => { + const onGrand = (id: string) => { if (!callUI) { callUI = new CallWindow(app.debug.error, this.options.callUITemplate) } @@ -203,7 +217,7 @@ export default class Assist { annot.mount() return callingAgents.get(id) } - const onRelease = (id, isDenied) => { + const onRelease = (id?: string | null, isDenied?: boolean) => { { if (id) { const cb = this.agents[id].onControlReleased @@ -237,7 +251,7 @@ export default class Assist { const onAcceptRecording = () => { socket.emit('recording_accepted') } - const onRejectRecording = (agentData) => { + const onRejectRecording = (agentData: AgentInfo) => { socket.emit('recording_rejected') this.options.onRecordingDeny?.(agentData || {}) @@ -276,7 +290,7 @@ export default class Assist { socket.on('startAnnotation', (id, event) => processEvent(id, event, (_, d) => annot?.start(d))) socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop)) - socket.on('NEW_AGENT', (id: string, info) => { + socket.on('NEW_AGENT', (id: string, info: AgentInfo) => { this.agents[id] = { onDisconnect: this.options.onAgentConnect?.(info), agentInfo: info, // TODO ? @@ -386,7 +400,7 @@ export default class Assist { host: this.getHost(), path: this.getBasePrefixUrl()+'/assist', port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443, - //debug: appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. + debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. } if (this.options.config) { peerOptions['config'] = this.options.config @@ -547,6 +561,38 @@ export default class Assist { app.debug.log(reason) }) }) + + app.nodes.attachNodeCallback((node) => { + const id = app.nodes.getID(node) + if (id && hasTag(node, 'canvas')) { + const canvasPId = `${app.getProjectKey()}-${sessionId}-${id}` + if (!this.canvasPeer) this.canvasPeer = new safeCastedPeer(canvasPId, peerOptions) as Peer + const canvasHandler = new Canvas( + node as unknown as HTMLCanvasElement, + id, + 30, + (stream: MediaStream) => { + Object.values(this.agents).forEach(agent => { + if (agent.agentInfo) { + const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas` + const connection = this.canvasPeer?.connect(target) + connection?.on('open', () => { + if (agent.agentInfo) { + const pCall = this.canvasPeer?.call(target, stream) + pCall?.on('error', app.debug.error) + } + }) + connection?.on('error', app.debug.error) + this.canvasPeer?.on('error', app.debug.error) + } else { + app.debug.error('Assist: cant establish canvas peer to agent, no agent info') + } + }) + }, + ) + this.canvasMap.set(id, canvasHandler) + } + }) } private playNotificationSound() { @@ -572,3 +618,16 @@ export default class Assist { } } } + +/** simple peers impl + * const slPeer = new SLPeer({ initiator: true, stream: stream, }) + * // slPeer.on('signal', (data: any) => { + * // this.emit('c_signal', { data, id, }) + * // }) + * // this.socket?.on('c_signal', (tab: string, data: any) => { + * // console.log(data) + * // slPeer.signal(data) + * // }) + * // slPeer.on('error', console.error) + * // this.emit('canvas_stream', { canvasId, }) + * */ \ No newline at end of file diff --git a/tracker/tracker-assist/src/Canvas.ts b/tracker/tracker-assist/src/Canvas.ts new file mode 100644 index 000000000..2faba0a09 --- /dev/null +++ b/tracker/tracker-assist/src/Canvas.ts @@ -0,0 +1,61 @@ + +export default class CanvasRecorder { + stream: MediaStream | null; + + constructor( + private readonly canvas: HTMLCanvasElement, + private readonly canvasId: number, + private readonly fps: number, + private readonly onStream: (stream: MediaStream) => void + ) { + this.canvas.getContext('2d', { alpha: true, }) + const stream = this.canvas.captureStream(this.fps) + this.emitStream(stream) + } + + restart() { + // this.stop() + const stream = this.canvas.captureStream(this.fps) + this.stream = stream + this.emitStream(stream) + } + + toggleLocal(stream: MediaStream) { + const possibleVideoEl = document.getElementById('canvas-or-testing') + if (possibleVideoEl) { + document.body.removeChild(possibleVideoEl) + } + const video = document.createElement('video') + video.width = 520 + video.height = 400 + video.id = 'canvas-or-testing' + video.setAttribute('autoplay', 'true') + video.setAttribute('muted', 'true') + video.setAttribute('playsinline', 'true') + video.crossOrigin = 'anonymous' + document.body.appendChild(video) + + video.srcObject = stream + + void video.play() + video.addEventListener('error', (e) => { + console.error('Video error:', e) + }) + } + + emitStream(stream?: MediaStream) { + if (stream) { + return this.onStream(stream) + } + if (this.stream) { + this.onStream(this.stream) + } else { + console.error('no stream for canvas', this.canvasId) + } + } + + stop() { + this.stream?.getTracks().forEach((track) => track.stop()) + this.stream = null + } +} diff --git a/tracker/tracker-assist/src/version.ts b/tracker/tracker-assist/src/version.ts index 819bfe0a6..cfebd79bf 100644 --- a/tracker/tracker-assist/src/version.ts +++ b/tracker/tracker-assist/src/version.ts @@ -1 +1 @@ -export const pkgVersion = '6.0.3' +export const pkgVersion = '6.0.4-57' diff --git a/tracker/tracker/bun.lockb b/tracker/tracker/bun.lockb index b4c48d4f8..8e730cd9e 100755 Binary files a/tracker/tracker/bun.lockb and b/tracker/tracker/bun.lockb differ diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 8bc3c6fba..14c630fae 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,14 +1,15 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "10.0.2", + "version": "10.0.3-43", "keywords": [ "logging", "replay" ], "author": "Alex Tsokurov", "contributors": [ - "Aleksandr K " + "Aleksandr K ", + "Nikita D " ], "license": "MIT", "type": "module", diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 792b27888..46eda477e 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -71,6 +71,7 @@ export declare const enum Type { ResourceTiming = 116, TabChange = 117, TabData = 118, + CanvasNode = 119, } @@ -562,6 +563,12 @@ export type TabData = [ /*tabId:*/ string, ] +export type CanvasNode = [ + /*type:*/ Type.CanvasNode, + /*nodeId:*/ string, + /*timestamp:*/ number, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequestDeprecated | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingDeprecated | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | NetworkRequest | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming | TabChange | TabData + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequestDeprecated | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingDeprecated | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | NetworkRequest | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming | TabChange | TabData | CanvasNode export default Message diff --git a/tracker/tracker/src/main/app/canvas.ts b/tracker/tracker/src/main/app/canvas.ts new file mode 100644 index 000000000..3163ea097 --- /dev/null +++ b/tracker/tracker/src/main/app/canvas.ts @@ -0,0 +1,130 @@ +import App from '../app/index.js' +import { hasTag } from './guards.js' +import Message, { CanvasNode } from './messages.gen.js' + +interface CanvasSnapshot { + images: { data: string; id: number }[] + createdAt: number +} + +interface Options { + fps: number + quality: 'low' | 'medium' | 'high' +} + +class CanvasRecorder { + private snapshots: Record = {} + private readonly intervals: NodeJS.Timeout[] = [] + private readonly interval: number + + constructor( + private readonly app: App, + private readonly options: Options, + ) { + this.interval = 1000 / options.fps + } + + startTracking() { + this.app.nodes.attachNodeCallback((node: Node): void => { + const id = this.app.nodes.getID(node) + if (!id || !hasTag(node, 'canvas') || this.snapshots[id]) { + return + } + const ts = this.app.timestamp() + this.snapshots[id] = { + images: [], + createdAt: ts, + } + const canvasMsg = CanvasNode(id.toString(), ts) + this.app.send(canvasMsg as Message) + const int = setInterval(() => { + const cid = this.app.nodes.getID(node) + const canvas = cid ? this.app.nodes.getNode(cid) : undefined + if (!canvas || !hasTag(canvas, 'canvas') || canvas !== node) { + console.log('Canvas element not in sync') + clearInterval(int) + } else { + const snapshot = captureSnapshot(canvas, this.options.quality) + this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot }) + if (this.snapshots[id].images.length > 9) { + this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt) + this.snapshots[id].images = [] + } + } + }, this.interval) + this.intervals.push(int) + }) + } + + sendSnaps(images: { data: string; id: number }[], canvasId: number, createdAt: number) { + if (Object.keys(this.snapshots).length === 0) { + console.log(this.snapshots) + return + } + const formData = new FormData() + images.forEach((snapshot) => { + const blob = dataUrlToBlob(snapshot.data)[0] + formData.append('snapshot', blob, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`) + // saveImageData(snapshot.data, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`) + }) + + fetch(this.app.options.ingestPoint + '/v1/web/images', { + method: 'POST', + headers: { + Authorization: `Bearer ${this.app.session.getSessionToken() ?? ''}`, + }, + body: formData, + }) + .then((r) => { + console.log('done', r) + }) + .catch((e) => { + console.error('error saving canvas', e) + }) + } + + clear() { + console.log('cleaning up') + this.intervals.forEach((int) => clearInterval(int)) + this.snapshots = {} + } +} + +const qualityInt = { + low: 0.33, + medium: 0.55, + high: 0.8, +} + +function captureSnapshot(canvas: HTMLCanvasElement, quality: 'low' | 'medium' | 'high' = 'medium') { + const imageFormat = 'image/jpeg' // or /png' + return canvas.toDataURL(imageFormat, qualityInt[quality]) +} + +function dataUrlToBlob(dataUrl: string): [Blob, Uint8Array] { + const [header, base64] = dataUrl.split(',') + // @ts-ignore + const mime = header.match(/:(.*?);/)[1] + const blobStr = atob(base64) + let n = blobStr.length + const u8arr = new Uint8Array(n) + + while (n--) { + u8arr[n] = blobStr.charCodeAt(n) + } + + return [new Blob([u8arr], { type: mime }), u8arr] +} + +function saveImageData(imageDataUrl: string, name: string) { + const link = document.createElement('a') + link.href = imageDataUrl + link.download = name + link.style.display = 'none' + + document.body.appendChild(link) + link.click() + document.body.removeChild(link) +} + +export default CanvasRecorder diff --git a/tracker/tracker/src/main/app/guards.ts b/tracker/tracker/src/main/app/guards.ts index 5379f9387..11edb1dd1 100644 --- a/tracker/tracker/src/main/app/guards.ts +++ b/tracker/tracker/src/main/app/guards.ts @@ -38,6 +38,7 @@ type TagTypeMap = { iframe: HTMLIFrameElement style: HTMLStyleElement | SVGStyleElement link: HTMLLinkElement + canvas: HTMLCanvasElement } export function hasTag( el: Node, diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 13499a669..068e1dabf 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -23,6 +23,7 @@ import type { Options as SanitizerOptions } from './sanitizer.js' import type { Options as LoggerOptions } from './logger.js' import type { Options as SessOptions } from './session.js' import type { Options as NetworkOptions } from '../modules/network.js' +import CanvasRecorder from './canvas.js' import type { Options as WebworkerOptions, @@ -142,6 +143,12 @@ export default class App { private readonly bc: BroadcastChannel | null = null private readonly contextId public attributeSender: AttributeSender + private canvasRecorder: CanvasRecorder | null = null + private canvasOptions = { + canvasEnabled: false, + canvasQuality: 'medium', + canvasFPS: 1, + } constructor(projectKey: string, sessionToken: string | undefined, options: Partial) { // if (options.onStart !== undefined) { @@ -631,6 +638,9 @@ export default class App { userDevice, userOS, userState, + canvasEnabled, + canvasQuality, + canvasFPS, } = r if ( typeof token !== 'string' || @@ -675,13 +685,19 @@ export default class App { }) this.compressionThreshold = compressionThreshold - const onStartInfo = { sessionToken: token, userUUID, sessionID } // TODO: start as early as possible (before receiving the token) this.startCallbacks.forEach((cb) => cb(onStartInfo)) // MBTODO: callbacks after DOM "mounted" (observed) this.observer.observe() this.ticker.start() + + if (canvasEnabled) { + this.canvasRecorder = + this.canvasRecorder ?? + new CanvasRecorder(this, { fps: canvasFPS, quality: canvasQuality }) + this.canvasRecorder.startTracking() + } this.activityState = ActivityState.Active this.notify.log('OpenReplay tracking started.') @@ -752,6 +768,7 @@ export default class App { if (this.worker && stopWorker) { this.worker.postMessage('stop') } + this.canvasRecorder?.clear() } finally { this.activityState = ActivityState.NotActive } diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index 2e3199248..f186a5f27 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -912,3 +912,14 @@ export function TabData( ] } +export function CanvasNode( + nodeId: string, + timestamp: number, +): Messages.CanvasNode { + return [ + Messages.Type.CanvasNode, + nodeId, + timestamp, + ] +} + diff --git a/tracker/tracker/src/tests/guards.test.ts b/tracker/tracker/src/tests/guards.test.ts new file mode 100644 index 000000000..c34775db1 --- /dev/null +++ b/tracker/tracker/src/tests/guards.test.ts @@ -0,0 +1,71 @@ +import { describe, beforeEach, expect, test } from '@jest/globals' +import { + isNode, + isSVGElement, + isElementNode, + isCommentNode, + isTextNode, + isDocument, + isRootNode, + hasTag, +} from '../main/app/guards' + +describe('DOM utility functions', () => { + let elementNode: Element + let commentNode: Comment + let textNode: Text + let documentNode: Document + let fragmentNode: DocumentFragment + let svgElement: SVGElement + + beforeEach(() => { + elementNode = document.createElement('div') + commentNode = document.createComment('This is a comment') + textNode = document.createTextNode('This is text') + documentNode = document + fragmentNode = document.createDocumentFragment() + svgElement = document.createElementNS('http://www.w3.org/2000/svg', 'svg') + }) + + test('isNode', () => { + expect(isNode(elementNode)).toBeTruthy() + expect(isNode(null)).toBeFalsy() + }) + + test('isSVGElement', () => { + expect(isSVGElement(svgElement)).toBeTruthy() + expect(isSVGElement(elementNode)).toBeFalsy() + }) + + test('isElementNode', () => { + expect(isElementNode(elementNode)).toBeTruthy() + expect(isElementNode(textNode)).toBeFalsy() + }) + + test('isCommentNode', () => { + expect(isCommentNode(commentNode)).toBeTruthy() + expect(isCommentNode(elementNode)).toBeFalsy() + }) + + test('isTextNode', () => { + expect(isTextNode(textNode)).toBeTruthy() + expect(isTextNode(elementNode)).toBeFalsy() + }) + + test('isDocument', () => { + expect(isDocument(documentNode)).toBeTruthy() + expect(isDocument(elementNode)).toBeFalsy() + }) + + test('isRootNode', () => { + expect(isRootNode(documentNode)).toBeTruthy() + expect(isRootNode(fragmentNode)).toBeTruthy() + expect(isRootNode(elementNode)).toBeFalsy() + }) + + test('hasTag', () => { + const imgElement = document.createElement('img') + expect(hasTag(imgElement, 'img')).toBeTruthy() + expect(hasTag(elementNode, 'img')).toBeFalsy() + }) +}) diff --git a/tracker/tracker/src/main/app/nodes.unit.test.ts b/tracker/tracker/src/tests/nodes.unit.test.ts similarity index 81% rename from tracker/tracker/src/main/app/nodes.unit.test.ts rename to tracker/tracker/src/tests/nodes.unit.test.ts index 8b32f7916..213b83b6b 100644 --- a/tracker/tracker/src/main/app/nodes.unit.test.ts +++ b/tracker/tracker/src/tests/nodes.unit.test.ts @@ -1,5 +1,5 @@ -import Nodes from './nodes' -import { describe, beforeEach, expect, it, jest } from '@jest/globals' +import Nodes from '../main/app/nodes' +import { describe, beforeEach, expect, test, jest } from '@jest/globals' describe('Nodes', () => { let nodes: Nodes @@ -11,13 +11,13 @@ describe('Nodes', () => { mockCallback.mockClear() }) - it('attachNodeCallback', () => { + test('attachNodeCallback', () => { nodes.attachNodeCallback(mockCallback) nodes.callNodeCallbacks(document.createElement('div'), true) expect(mockCallback).toHaveBeenCalled() }) - it('attachNodeListener is listening to events', () => { + test('attachNodeListener is listening to events', () => { const node = document.createElement('div') const mockListener = jest.fn() document.body.appendChild(node) @@ -26,7 +26,7 @@ describe('Nodes', () => { node.dispatchEvent(new Event('click')) expect(mockListener).toHaveBeenCalled() }) - it('attachNodeListener is calling native method', () => { + test('attachNodeListener is calling native method', () => { const node = document.createElement('div') const mockListener = jest.fn() const addEventListenerSpy = jest.spyOn(node, 'addEventListener') @@ -36,55 +36,55 @@ describe('Nodes', () => { expect(addEventListenerSpy).toHaveBeenCalledWith('click', mockListener, true) }) - it('registerNode', () => { + test('registerNode', () => { const node = document.createElement('div') const [id, isNew] = nodes.registerNode(node) expect(id).toBeDefined() expect(isNew).toBe(true) }) - it('unregisterNode', () => { + test('unregisterNode', () => { const node = document.createElement('div') const [id] = nodes.registerNode(node) const unregisteredId = nodes.unregisterNode(node) expect(unregisteredId).toBe(id) }) - it('cleanTree', () => { + test('cleanTree', () => { const node = document.createElement('div') nodes.registerNode(node) nodes.cleanTree() expect(nodes.getNodeCount()).toBe(0) }) - it('callNodeCallbacks', () => { + test('callNodeCallbacks', () => { nodes.attachNodeCallback(mockCallback) const node = document.createElement('div') nodes.callNodeCallbacks(node, true) expect(mockCallback).toHaveBeenCalledWith(node, true) }) - it('getID', () => { + test('getID', () => { const node = document.createElement('div') const [id] = nodes.registerNode(node) const fetchedId = nodes.getID(node) expect(fetchedId).toBe(id) }) - it('getNode', () => { + test('getNode', () => { const node = document.createElement('div') const [id] = nodes.registerNode(node) const fetchedNode = nodes.getNode(id) expect(fetchedNode).toBe(node) }) - it('getNodeCount', () => { + test('getNodeCount', () => { expect(nodes.getNodeCount()).toBe(0) nodes.registerNode(document.createElement('div')) expect(nodes.getNodeCount()).toBe(1) }) - it('clear', () => { + test('clear', () => { nodes.registerNode(document.createElement('div')) nodes.clear() expect(nodes.getNodeCount()).toBe(0) diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index a66283322..4ae5cfbd3 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -286,6 +286,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.string(msg[1]) break + case Messages.Type.CanvasNode: + return this.string(msg[1]) && this.uint(msg[2]) + break + } }