jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

spot: mix network requests with webRequest data for better tracking

Browse source
This commit is contained in:
nick-delirium 2025-01-09 14:17:49 +01:00 • committed by Delirium
parent 28dea3b225
commit 8ba35b1324
5 changed files with 195 additions and 185 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
spot/entrypoints/background.ts
View file

@ -1,4 +1,10 @@
import { isTokenExpired } from "~/utils/jwt"; import { isTokenExpired } from "~/utils/jwt";
import {
startTrackingNetwork,
getFinalRequests,
stopTrackingNetwork,
} from "~/utils/networkTracking";
import { mergeRequests } from "~/utils/networkTrackingUtils";
let checkBusy = false; let checkBusy = false;
@ -136,6 +142,7 @@ export default defineBackground(() => {
let finalVideoBase64 = ""; let finalVideoBase64 = "";
let finalReady = false; let finalReady = false;
let finalSpotObj: SpotObj = defaultSpotObj; let finalSpotObj: SpotObj = defaultSpotObj;
let injectNetworkRequests = [];
let onStop: (() => void) | null = null; let onStop: (() => void) | null = null;
let settings = defaultSettings; let settings = defaultSettings;
let recordingState = { let recordingState = {
@ -339,6 +346,7 @@ export default defineBackground(() => {
recording: REC_STATE.stopped, recording: REC_STATE.stopped,
audioPerm: request.permissions ? (request.mic ? 2 : 1) : 0, audioPerm: request.permissions ? (request.mic ? 2 : 1) : 0,
}; };
startTrackingNetwork();
if (request.area === "tab") { if (request.area === "tab") {
browser.tabs browser.tabs
.query({ .query({
@ -577,7 +585,7 @@ export default defineBackground(() => {
return "pong"; return "pong";
} }
if (request.type === messages.injected.from.bumpNetwork) { if (request.type === messages.injected.from.bumpNetwork) {
finalSpotObj.network.push(request.event); injectNetworkRequests.push(request.event);
return "pong"; return "pong";
} }
if (request.type === messages.content.from.bumpClicks) { if (request.type === messages.content.from.bumpClicks) {
@ -649,7 +657,7 @@ export default defineBackground(() => {
title: "JS Error", title: "JS Error",
time: (l.time - finalSpotObj.startTs) / 1000, time: (l.time - finalSpotObj.startTs) / 1000,
})); }));
const network = finalSpotObj.network const network = [...injectNetworkRequests, ...finalSpotObj.network]
.filter((net) => net.statusCode >= 400 || net.error) .filter((net) => net.statusCode >= 400 || net.error)
.map((n) => ({ .map((n) => ({
title: "Network Error", title: "Network Error",
@ -665,8 +673,19 @@ export default defineBackground(() => {
} }
if (request.type === messages.content.from.toStop) { if (request.type === messages.content.from.toStop) {
if (recordingState.recording === REC_STATE.stopped) { if (recordingState.recording === REC_STATE.stopped) {
return console.error('Calling stopped recording?') return console.error("Calling stopped recording?");
} }
const networkRequests = getFinalRequests(
recordingState.activeTabId ?? false,
);
stopTrackingNetwork();
const mappedNetwork = mergeRequests(
networkRequests,
injectNetworkRequests,
);
injectNetworkRequests = [];
finalSpotObj.network = mappedNetwork;
browser.runtime browser.runtime
.sendMessage({ .sendMessage({
type: messages.offscreen.to.stopRecording, type: messages.offscreen.to.stopRecording,
@ -749,12 +768,12 @@ export default defineBackground(() => {
if (request.type === messages.content.from.saveSpotVidChunk) { if (request.type === messages.content.from.saveSpotVidChunk) {
finalVideoBase64 += request.part; finalVideoBase64 += request.part;
finalReady = request.index === request.total - 1; finalReady = request.index === request.total - 1;
const getPlatformData = async () => {
const vendor = await browser.runtime.getPlatformInfo();
const platform = `${vendor.os} ${vendor.arch}`;
return { platform };
};
if (finalReady) { if (finalReady) {
const getPlatformData = async () => {
const vendor = await browser.runtime.getPlatformInfo();
const platform = `${vendor.os} ${vendor.arch}`;
return { platform };
};
const duration = finalSpotObj.crop const duration = finalSpotObj.crop
? finalSpotObj.crop[1] - finalSpotObj.crop[0] ? finalSpotObj.crop[1] - finalSpotObj.crop[0]
: finalSpotObj.duration; : finalSpotObj.duration;
@ -1155,7 +1174,7 @@ export default defineBackground(() => {
if (state === REC_STATE.stopped) { if (state === REC_STATE.stopped) {
return stopNavListening(); return stopNavListening();
} }
contentArmy[details.tabId] = false contentArmy[details.tabId] = false;
if (area === "tab" && (!trackedTab || details.tabId !== trackedTab)) { if (area === "tab" && (!trackedTab || details.tabId !== trackedTab)) {
return; return;
@ -1179,10 +1198,10 @@ export default defineBackground(() => {
} }
function startNavListening() { function startNavListening() {
browser.webNavigation.onCompleted.addListener(tabNavigatedListener) browser.webNavigation.onCompleted.addListener(tabNavigatedListener);
} }
function stopNavListening() { function stopNavListening() {
browser.webNavigation.onCompleted.removeListener(tabNavigatedListener) browser.webNavigation.onCompleted.removeListener(tabNavigatedListener);
} }
/** discards recording if was recording single tab and its now closed */ /** discards recording if was recording single tab and its now closed */

8
spot/entrypoints/content/index.tsx
View file

@ -270,16 +270,16 @@ export default defineContentScript({
document.head.appendChild(scriptEl); document.head.appendChild(scriptEl);
} }
function startConsoleTracking() { function startConsoleTracking() {
injectScript() injectScript();
setTimeout(() => { setTimeout(() => {
window.postMessage({ type: "injected:c-start" }); window.postMessage({ type: "injected:c-start" });
}, 100); }, 100);
} }
function startNetworkTracking() { function startNetworkTracking() {
injectScript() injectScript();
setTimeout(() => { setTimeout(() => {
window.postMessage({ type: "injected:n-start" }); window.postMessage({ type: "injected:n-start" });
}, 100) }, 100);
} }
function stopConsoleTracking() { function stopConsoleTracking() {
@ -325,7 +325,7 @@ export default defineContentScript({
setInterval(() => { setInterval(() => {
void browser.runtime.sendMessage({ type: "ort:content-ready" }); void browser.runtime.sendMessage({ type: "ort:content-ready" });
}, 250) }, 250);
// @ts-ignore false positive // @ts-ignore false positive
browser.runtime.onMessage.addListener((message: any, resp) => { browser.runtime.onMessage.addListener((message: any, resp) => {
if (message.type === "content:mount") { if (message.type === "content:mount") {

325
spot/utils/networkTracking.ts
View file

@ -1,169 +1,156 @@
// import { import {
// SpotNetworkRequest, SpotNetworkRequest,
// filterBody, filterBody,
// filterHeaders, filterHeaders,
// tryFilterUrl, tryFilterUrl,
// TrackedRequest, TrackedRequest,
// } from "./networkTrackingUtils"; } from "./networkTrackingUtils";
//
// export const rawRequests: (TrackedRequest & { export const rawRequests: Array<
// startTs: number; TrackedRequest & { startTs: number; duration: number }
// duration: number; > = [];
// })[] = [];
// function getRequestStatus(request: TrackedRequest): number {
// export function createSpotNetworkRequestV1( if (request.statusCode) return request.statusCode;
// trackedRequest: TrackedRequest, if (request.error) return 0;
// trackedTab?: number, return 200;
// ) { }
// if (trackedRequest.tabId === -1) {
// return; function modifyOnSpot(request: TrackedRequest) {
// } const id = request.requestId;
// if (trackedTab && trackedTab !== trackedRequest.tabId) { const index = rawRequests.findIndex((r) => r.requestId === id);
// return; const ts = Date.now();
// } const start = rawRequests[index]?.startTs ?? ts;
// if ( rawRequests[index] = {
// ["ping", "beacon", "image", "script", "font"].includes(trackedRequest.type) ...rawRequests[index],
// ) { ...request,
// if (!trackedRequest.statusCode || trackedRequest.statusCode < 400) { duration: ts - start,
// return; };
// } }
// }
// const type = ["stylesheet", "script", "image", "media", "font"].includes( function trackOnBefore(
// trackedRequest.type, details: browser.webRequest._OnBeforeRequestDetails & { reqBody?: string },
// ) ) {
// ? "resource" if (details.method === "POST" && details.requestBody) {
// : trackedRequest.type; if (details.requestBody.formData) {
// details.reqBody = JSON.stringify(details.requestBody.formData);
// const requestHeaders = trackedRequest.requestHeaders } else if (details.requestBody.raw) {
// ? filterHeaders(trackedRequest.requestHeaders) const raw = details.requestBody.raw[0]?.bytes;
// : {}; if (raw) details.reqBody = new TextDecoder("utf-8").decode(raw);
// const responseHeaders = trackedRequest.responseHeaders }
// ? filterHeaders(trackedRequest.responseHeaders) }
// : {}; rawRequests.push({ ...details, startTs: Date.now(), duration: 0 });
// }
// const reqSize = trackedRequest.reqBody
// ? trackedRequest.requestSize || trackedRequest.reqBody.length function trackOnHeaders(
// : 0; details: browser.webRequest._OnBeforeSendHeadersDetails,
// ) {
// const status = getRequestStatus(trackedRequest); modifyOnSpot(details);
// let body; }
// if (trackedRequest.reqBody) {
// try { function trackOnCompleted(details: browser.webRequest._OnCompletedDetails) {
// body = filterBody(trackedRequest.reqBody); modifyOnSpot(details);
// } catch (e) { }
// body = "Error parsing body";
// console.error(e); function trackOnError(details: browser.webRequest._OnErrorOccurredDetails) {
// } modifyOnSpot(details);
// } else { }
// body = "";
// } // Build final SpotNetworkRequest objects
// const request: SpotNetworkRequest = { function createSpotNetworkRequest(
// method: trackedRequest.method, trackedRequest: TrackedRequest,
// type, trackedTab?: number,
// body, ): SpotNetworkRequest | undefined {
// responseBody: "", if (trackedRequest.tabId === -1) return;
// requestHeaders, if (trackedTab && trackedTab !== trackedRequest.tabId) return;
// responseHeaders,
// time: trackedRequest.timeStamp, if (
// statusCode: status, ["ping", "beacon", "image", "script", "font"].includes(trackedRequest.type)
// error: trackedRequest.error, ) {
// url: tryFilterUrl(trackedRequest.url), if (!trackedRequest.statusCode || trackedRequest.statusCode < 400) return;
// fromCache: trackedRequest.fromCache || false, }
// encodedBodySize: reqSize,
// responseBodySize: trackedRequest.responseSize, const type = ["stylesheet", "script", "image", "media", "font"].includes(
// duration: trackedRequest.duration, trackedRequest.type,
// }; )
// ? "resource"
// return request; : trackedRequest.type;
// }
// const requestHeaders = trackedRequest.requestHeaders
// function modifyOnSpot(request: TrackedRequest) { ? filterHeaders(trackedRequest.requestHeaders)
// const id = request.requestId; : {};
// const index = rawRequests.findIndex((r) => r.requestId === id); const responseHeaders = trackedRequest.responseHeaders
// const ts = Date.now(); ? filterHeaders(trackedRequest.responseHeaders)
// const start = rawRequests[index]?.startTs ?? ts; : {};
// rawRequests[index] = {
// ...rawRequests[index], const reqSize = trackedRequest.reqBody
// ...request, ? trackedRequest.requestSize || trackedRequest.reqBody.length
// duration: ts - start, : 0;
// }; const status = getRequestStatus(trackedRequest);
// }
// let body = "";
// const trackOnBefore = ( if (trackedRequest.reqBody) {
// details: WebRequest.OnBeforeRequestDetailsType & { reqBody: string }, try {
// ) => { body = filterBody(trackedRequest.reqBody);
// if (details.method === "POST" && details.requestBody) { } catch (e) {
// const requestBody = details.requestBody; body = "Error parsing body";
// if (requestBody.formData) { console.error(e);
// details.reqBody = JSON.stringify(requestBody.formData); }
// } else if (requestBody.raw) { }
// const raw = requestBody.raw[0]?.bytes;
// if (raw) { const request: SpotNetworkRequest = {
// details.reqBody = new TextDecoder("utf-8").decode(raw); method: trackedRequest.method,
// } type,
// } body,
// } responseBody: "",
// rawRequests.push({ ...details, startTs: Date.now(), duration: 0 }); requestHeaders,
// }; responseHeaders,
// const trackOnCompleted = (details: WebRequest.OnCompletedDetailsType) => { timestamp: trackedRequest.timeStamp,
// modifyOnSpot(details); statusCode: status,
// }; error: trackedRequest.error,
// const trackOnHeaders = (details: WebRequest.OnBeforeSendHeadersDetailsType) => { url: tryFilterUrl(trackedRequest.url),
// modifyOnSpot(details); fromCache: trackedRequest.fromCache || false,
// }; encodedBodySize: reqSize,
// const trackOnError = (details: WebRequest.OnErrorOccurredDetailsType) => { responseBodySize: trackedRequest.responseSize,
// modifyOnSpot(details); duration: trackedRequest.duration,
// }; };
// export function startTrackingNetwork() {
// rawRequests.length = 0; return request;
// browser.webRequest.onBeforeRequest.addListener( }
// // @ts-ignore
// trackOnBefore, export function startTrackingNetwork() {
// { urls: ["<all_urls>"] }, rawRequests.length = 0;
// ["requestBody"], browser.webRequest.onBeforeRequest.addListener(
// ); trackOnBefore,
// browser.webRequest.onBeforeSendHeaders.addListener( { urls: ["<all_urls>"] },
// trackOnHeaders, ["requestBody"], // allows capturing POST bodies
// { urls: ["<all_urls>"] }, );
// ["requestHeaders"], browser.webRequest.onBeforeSendHeaders.addListener(
// ); trackOnHeaders,
// browser.webRequest.onCompleted.addListener( { urls: ["<all_urls>"] },
// trackOnCompleted, ["requestHeaders"],
// { );
// urls: ["<all_urls>"], browser.webRequest.onCompleted.addListener(
// }, trackOnCompleted,
// ["responseHeaders"], { urls: ["<all_urls>"] },
// ); ["responseHeaders"],
// browser.webRequest.onErrorOccurred.addListener( );
// trackOnError, browser.webRequest.onErrorOccurred.addListener(trackOnError, {
// { urls: ["<all_urls>"],
// urls: ["<all_urls>"], });
// }, }
// ["extraHeaders"],
// ); export function stopTrackingNetwork() {
// } browser.webRequest.onBeforeRequest.removeListener(trackOnBefore);
// browser.webRequest.onBeforeSendHeaders.removeListener(trackOnHeaders);
// export function stopTrackingNetwork() { browser.webRequest.onCompleted.removeListener(trackOnCompleted);
// browser.webRequest.onBeforeRequest.removeListener(trackOnBefore); browser.webRequest.onErrorOccurred.removeListener(trackOnError);
// browser.webRequest.onCompleted.removeListener(trackOnCompleted); }
// browser.webRequest.onErrorOccurred.removeListener(trackOnError);
// } export function getFinalRequests(tabId?: number): SpotNetworkRequest[] {
// const finalRequests = rawRequests
// function getRequestStatus(request: any): number { .map((r) => createSpotNetworkRequest(r, tabId))
// if (request.statusCode) { .filter((r) => r !== undefined) as SpotNetworkRequest[];
// return request.statusCode; rawRequests.length = 0;
// } return finalRequests;
// if (request.error) { }
// return 0;
// }
// return 200;
// }
//
// export function getFinalRequests(tabId: number): SpotNetworkRequest[] {
// const finalRequests = rawRequests
// .map((r) => createSpotNetworkRequest(r, tabId))
// .filter((r) => r !== undefined);
// rawRequests.length = 0;
//
// return finalRequests;
// }

4
spot/utils/networkTrackingUtils.ts
View file

@ -82,7 +82,9 @@ export const sensitiveParams = new Set([
"account_key", "account_key",
]); ]);
export function filterHeaders(headers: Record<string, string> | { name: string; value: string }[]) { export function filterHeaders(
headers: Record<string, string> | { name: string; value: string }[],
) {
const filteredHeaders: Record<string, string> = {}; const filteredHeaders: Record<string, string> = {};
if (Array.isArray(headers)) { if (Array.isArray(headers)) {
headers.forEach(({ name, value }) => { headers.forEach(({ name, value }) => {

2
spot/wxt.config.ts
View file

@ -26,6 +26,8 @@ export default defineConfig({
"offscreen", "offscreen",
"unlimitedStorage", "unlimitedStorage",
"webNavigation", "webNavigation",
"webRequest",
"<all_urls>",
], ],
}, },
}); });