From 2377cc79d005d9602907377595cf1e9ff6827f97 Mon Sep 17 00:00:00 2001 From: Delirium Date: Mon, 11 Mar 2024 14:26:50 +0100 Subject: [PATCH] feat(ui): switch canvas player to images (#1925) * feat(ui): switch canvas player to images * feat(ui): complete player * fix(ui): expect undefined .tar for canvas * fix(ui): improve error handling * fix(ui): file format --- .../components/CaptureRate.tsx | 2 +- frontend/app/mstore/types/FeatureFlag.ts | 4 +- frontend/app/mstore/types/filter.ts | 4 +- frontend/app/mstore/types/filterItem.ts | 16 +- frontend/app/player/web/MessageLoader.ts | 2 +- frontend/app/player/web/TabManager.ts | 12 +- .../app/player/web/managers/CanvasManager.ts | 165 +++++++++++++++--- 7 files changed, 171 insertions(+), 34 deletions(-) diff --git a/frontend/app/components/shared/SessionSettings/components/CaptureRate.tsx b/frontend/app/components/shared/SessionSettings/components/CaptureRate.tsx index 2d19acef6..a8101215e 100644 --- a/frontend/app/components/shared/SessionSettings/components/CaptureRate.tsx +++ b/frontend/app/components/shared/SessionSettings/components/CaptureRate.tsx @@ -42,7 +42,7 @@ function CaptureRate(props: Props) { }, [projectId]); React.useEffect(() => { - setConditions(captureConditions.map((condition: any) => new Conditions(condition, true))); + setConditions(captureConditions.map((condition: any) => new Conditions(condition, true, isMobile))); }, [captureConditions]); const onCaptureRateChange = (input: string) => { diff --git a/frontend/app/mstore/types/FeatureFlag.ts b/frontend/app/mstore/types/FeatureFlag.ts index eb81cc193..7e4d9c327 100644 --- a/frontend/app/mstore/types/FeatureFlag.ts +++ b/frontend/app/mstore/types/FeatureFlag.ts @@ -7,12 +7,12 @@ export class Conditions { filter = new Filter().fromJson({ name: 'Rollout conditions', filters: [] }); name = 'Condition Set'; - constructor(data?: Record, isConditional?: boolean) { + constructor(data?: Record, isConditional?: boolean, isMobile?: boolean) { makeAutoObservable(this); this.name = data?.name; if (data && (data.rolloutPercentage || data.captureRate)) { this.rolloutPercentage = data.rolloutPercentage ?? data.captureRate; - this.filter = new Filter(isConditional).fromJson(data); + this.filter = new Filter(isConditional, isMobile).fromJson(data); } } diff --git a/frontend/app/mstore/types/filter.ts b/frontend/app/mstore/types/filter.ts index 67dad77c7..81dc1a2ea 100644 --- a/frontend/app/mstore/types/filter.ts +++ b/frontend/app/mstore/types/filter.ts @@ -16,7 +16,7 @@ export default class Filter { page: number = 1 limit: number = 10 - constructor(private readonly isConditional = false) { + constructor(private readonly isConditional = false, private readonly isMobile = false) { makeAutoObservable(this, { filters: observable, eventsOrder: observable, @@ -64,7 +64,7 @@ export default class Filter { fromJson(json: any) { this.name = json.name this.filters = json.filters.map((i: Record) => - new FilterItem(undefined, this.isConditional).fromJson(i) + new FilterItem(undefined, this.isConditional, this.isMobile).fromJson(i) ); this.eventsOrder = json.eventsOrder return this diff --git a/frontend/app/mstore/types/filterItem.ts b/frontend/app/mstore/types/filterItem.ts index 34c2560cb..e32ebd7ef 100644 --- a/frontend/app/mstore/types/filterItem.ts +++ b/frontend/app/mstore/types/filterItem.ts @@ -1,6 +1,6 @@ import { makeAutoObservable, observable, action } from 'mobx'; -import { FilterKey, FilterType, FilterCategory } from 'Types/filter/filterType'; -import { filtersMap, conditionalFiltersMap } from 'Types/filter/newFilter'; +import { FilterKey, FilterType, FilterCategory } from 'Types/filter/filterType'; +import { filtersMap, conditionalFiltersMap, mobileConditionalFiltersMap } from 'Types/filter/newFilter'; export default class FilterItem { type: string = ''; @@ -21,7 +21,11 @@ export default class FilterItem { completed: number = 0; dropped: number = 0; - constructor(data: any = {}, private readonly isConditional?: boolean) { + constructor( + data: any = {}, + private readonly isConditional?: boolean, + private readonly isMobile?: boolean +) { makeAutoObservable(this, { type: observable, key: observable, @@ -61,7 +65,11 @@ export default class FilterItem { const isMetadata = json.type === FilterKey.METADATA; let _filter: any = (isMetadata ? filtersMap['_' + json.source] : filtersMap[json.type]) || {}; if (this.isConditional) { - _filter = conditionalFiltersMap[json.type] || conditionalFiltersMap[json.source]; + if (this.isMobile) { + _filter = mobileConditionalFiltersMap[json.type] || mobileConditionalFiltersMap[json.source]; + } else { + _filter = conditionalFiltersMap[json.type] || conditionalFiltersMap[json.source]; + } } if (mainFilterKey) { const mainFilter = filtersMap[mainFilterKey]; diff --git a/frontend/app/player/web/MessageLoader.ts b/frontend/app/player/web/MessageLoader.ts index 01585b6e4..a12281a59 100644 --- a/frontend/app/player/web/MessageLoader.ts +++ b/frontend/app/player/web/MessageLoader.ts @@ -4,9 +4,9 @@ import MFileReader from './messages/MFileReader'; import { loadFiles, requestEFSDom, requestEFSDevtools, requestTarball } from './network/loadFiles'; import logger from 'App/logger'; import unpack from 'Player/common/unpack'; +import unpackTar from 'Player/common/tarball'; import MessageManager from 'Player/web/MessageManager'; import IOSMessageManager from 'Player/mobile/IOSMessageManager'; -import unpackTar from 'Player/common/tarball'; interface State { firstFileLoading: boolean; diff --git a/frontend/app/player/web/TabManager.ts b/frontend/app/player/web/TabManager.ts index 9e336b7e3..c26f6a672 100644 --- a/frontend/app/player/web/TabManager.ts +++ b/frontend/app/player/web/TabManager.ts @@ -169,13 +169,19 @@ export default class TabSessionManager { case MType.CanvasNode: const managerId = `${msg.timestamp}_${msg.nodeId}`; if (!this.canvasManagers[managerId]) { - const filename = `${managerId}.mp4`; + const fileId = managerId; const delta = msg.timestamp - this.sessionStart; - const fileUrl = this.session.canvasURL.find((url: string) => url.includes(filename)); + const canvasNodeLinks = this.session.canvasURL.filter((url: string) => url.includes(fileId)) as string[]; + const tarball = canvasNodeLinks.find((url: string) => url.includes('.tar.')); + const mp4file = canvasNodeLinks.find((url: string) => url.includes('.mp4')); + if (!tarball && !mp4file) { + console.error('no canvas recording provided') + break; + } const manager = new CanvasManager( msg.nodeId, delta, - fileUrl, + [tarball, mp4file], this.getNode as (id: number) => VElement | undefined ); this.canvasManagers[managerId] = { manager, start: msg.timestamp, running: false }; diff --git a/frontend/app/player/web/managers/CanvasManager.ts b/frontend/app/player/web/managers/CanvasManager.ts index 4d60a1bae..57d05f499 100644 --- a/frontend/app/player/web/managers/CanvasManager.ts +++ b/frontend/app/player/web/managers/CanvasManager.ts @@ -1,9 +1,26 @@ -import { VElement } from "Player/web/managers/DOM/VirtualDOM"; +import { TarFile } from 'js-untar'; +import ListWalker from 'Player/common/ListWalker'; +import { VElement } from 'Player/web/managers/DOM/VirtualDOM'; +import unpack from 'Player/common/unpack'; +import unpackTar from 'Player/common/tarball'; -export default class CanvasManager { +const playMode = { + video: 'video', + snaps: 'snaps', +} as const; + +const TAR_MISSING = 'TAR_404'; +const MP4_MISSING = 'MP4_404'; + +type Timestamp = { time: number }; + +export default class CanvasManager extends ListWalker { private fileData: string | undefined; - private videoTag = document.createElement('video') + private videoTag = document.createElement('video'); + private snapImage = document.createElement('img'); private lastTs = 0; + private playMode: string = playMode.snaps; + private snapshots: Record = {}; constructor( /** @@ -14,21 +31,109 @@ export default class CanvasManager { * time between node creation and session start */ private readonly delta: number, - private readonly filename: string, - private readonly getNode: (id: number) => VElement | undefined) { - // getting mp4 file composed of canvas snapshot images - fetch(this.filename).then((r) => { - if (r.status === 200) { - r.blob().then((blob) => { - this.fileData = URL.createObjectURL(blob); - }) + private readonly links: [tar?: string, mp4?: string], + private readonly getNode: (id: number) => VElement | undefined + ) { + super(); + console.log(links); + // first we try to grab tar, then fallback to mp4 + this.loadTar() + .then((fileArr) => { + this.mapToSnapshots(fileArr); + }) + .catch((e) => { + if (e === TAR_MISSING && this.links[1]) { + this.loadMp4().catch((e2) => { + if (e2 === MP4_MISSING) { + return console.error( + `both tar and mp4 recordings for canvas ${this.nodeId} not found` + ); + } else { + return console.error('Failed to load canvas recording'); + } + }); } else { - return Promise.reject(`File ${this.filename} not found`) + return console.error('Failed to load canvas recording for node', this.nodeId); } - }).catch(console.error) + }); } + public mapToSnapshots(files: TarFile[]) { + const tempArr: Timestamp[] = []; + const filenameRegexp = /(\d+)_(\d+)_(\d+)\.jpeg$/; + const firstPair = files[0].name.match(filenameRegexp); + if (!firstPair) { + console.error('Invalid file name format', files[0].name); + return; + } + const sessionStart = firstPair ? parseInt(firstPair[1], 10) : 0; + files.forEach((file) => { + const [_, _1, _2, imageTimestampStr] = file.name.match(filenameRegexp) ?? [0, 0, 0, '0']; + + const imageTimestamp = parseInt(imageTimestampStr, 10); + + const messageTime = imageTimestamp - sessionStart; + this.snapshots[messageTime] = file; + tempArr.push({ time: messageTime }); + }); + + tempArr + .sort((a, b) => a.time - b.time) + .forEach((msg) => { + this.append(msg); + }); + } + + loadTar = async () => { + if (!this.links[0]) { + return Promise.reject(TAR_MISSING); + } + return fetch(this.links[0]) + .then((r) => { + if (r.status === 200) { + return r.arrayBuffer(); + } else { + return Promise.reject(TAR_MISSING); + } + }) + .then((buf) => { + const tar = unpack(new Uint8Array(buf)); + this.playMode = playMode.snaps; + return unpackTar(tar); + }); + }; + + loadMp4 = async () => { + if (!this.links[1]) { + return Promise.reject(MP4_MISSING); + } + return fetch(this.links[1]) + .then((r) => { + if (r.status === 200) { + return r.blob(); + } else { + return Promise.reject(MP4_MISSING); + } + }) + .then((blob) => { + this.playMode = playMode.video; + this.fileData = URL.createObjectURL(blob); + }); + }; + startVideo = () => { + if (this.playMode === playMode.snaps) { + this.snapImage.onload = () => { + const node = this.getNode(parseInt(this.nodeId, 10)); + if (node && node.node) { + const canvasCtx = (node.node as HTMLCanvasElement).getContext('2d'); + const canvasEl = node.node as HTMLVideoElement; + canvasCtx?.drawImage(this.snapImage, 0, 0, canvasEl.width, canvasEl.height); + } else { + console.error(`CanvasManager: Node ${this.nodeId} not found`); + } + }; + } if (!this.fileData) return; this.videoTag.setAttribute('autoplay', 'true'); this.videoTag.setAttribute('muted', 'true'); @@ -36,25 +141,43 @@ export default class CanvasManager { this.videoTag.setAttribute('crossorigin', 'anonymous'); this.videoTag.src = this.fileData; this.videoTag.currentTime = 0; - } + }; move(t: number) { + if (this.playMode === playMode.video) { + this.moveReadyVideo(t); + } else { + this.moveReadySnap(t); + } + } + + moveReadyVideo = (t: number) => { if (Math.abs(t - this.lastTs) < 100) return; this.lastTs = t; - const playTime = t - this.delta + const playTime = t - this.delta; if (playTime > 0) { - const node = this.getNode(parseInt(this.nodeId, 10)) + const node = this.getNode(parseInt(this.nodeId, 10)); if (node && node.node) { const canvasCtx = (node.node as HTMLCanvasElement).getContext('2d'); const canvasEl = node.node as HTMLVideoElement; if (!this.videoTag.paused) { - void this.videoTag.pause() + void this.videoTag.pause(); } - this.videoTag.currentTime = playTime/1000; + this.videoTag.currentTime = playTime / 1000; canvasCtx?.drawImage(this.videoTag, 0, 0, canvasEl.width, canvasEl.height); } else { - console.error(`CanvasManager: Node ${this.nodeId} not found`) + console.error(`CanvasManager: Node ${this.nodeId} not found`); } } - } -} \ No newline at end of file + }; + + moveReadySnap = (t: number) => { + const msg = this.moveGetLast(t); + if (msg) { + const file = this.snapshots[msg.time]; + if (file) { + this.snapImage.src = file.getBlobUrl(); + } + } + }; +}