From 373d71e4f3995934ab3a542b27d8941ff5bf0615 Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 18 Apr 2025 15:31:55 +0200 Subject: [PATCH] feat(ch-connector): added current url for all events --- backend/pkg/db/clickhouse/connector.go | 89 ++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/backend/pkg/db/clickhouse/connector.go b/backend/pkg/db/clickhouse/connector.go index 25ebc8a1e..44ee1c5c4 100644 --- a/backend/pkg/db/clickhouse/connector.go +++ b/backend/pkg/db/clickhouse/connector.go @@ -108,15 +108,15 @@ func (c *connectorImpl) newBatch(name, query string) error { var batches = map[string]string{ "sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, platform, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?)", "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, SUBSTR(?, 1, 8000))", - "pages": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - "clicks": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - "inputs": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - "errors": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", error_id, "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - "performance": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - "requests": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - "custom": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - "graphql": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - "issuesEvents": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", issue_type, issue_id, "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "pages": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "clicks": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "inputs": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "errors": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", error_id, "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "performance": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "requests": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "custom": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "graphql": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "issuesEvents": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$os", "$browser", "$referrer", "$country", "$state", "$city", "$current_url", issue_type, issue_id, "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, "issues": "INSERT INTO experimental.issues (project_id, issue_id, type, context_string) VALUES (?, ?, ?, ?)", "mobile_sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, platform, timezone) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?)", "mobile_custom": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, @@ -267,6 +267,13 @@ func (c *connectorImpl) InsertWebInputDuration(session *sessions.Session, msg *m true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, + cropString(msg.Url), nullableUint16(uint16(msg.InputDuration)), jsonString, ); err != nil { @@ -307,9 +314,15 @@ func (c *connectorImpl) InsertMouseThrashing(session *sessions.Session, msg *mes true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, + cropString(msg.Url), "mouse_thrashing", issueID, - cropString(msg.Url), jsonString, ); err != nil { c.checkError("issuesEvents", err) @@ -364,9 +377,15 @@ func (c *connectorImpl) InsertIssue(session *sessions.Session, msg *messages.Iss true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, + cropString(msg.Url), msg.Type, issueID, - cropString(msg.Url), jsonString, ); err != nil { c.checkError("issuesEvents", err) @@ -454,6 +473,12 @@ func (c *connectorImpl) InsertWebPageEvent(session *sessions.Session, msg *messa true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, cropString(msg.URL), jsonString, ); err != nil { @@ -514,6 +539,12 @@ func (c *connectorImpl) InsertWebClickEvent(session *sessions.Session, msg *mess true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, cropString(msg.Url), jsonString, ); err != nil { @@ -553,8 +584,14 @@ func (c *connectorImpl) InsertWebErrorEvent(session *sessions.Session, msg *type true, session.Platform, session.UserOSVersion, - msgID, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, cropString(msg.Url), + msgID, jsonString, ); err != nil { c.checkError("errors", err) @@ -604,6 +641,13 @@ func (c *connectorImpl) InsertWebPerformanceTrackAggr(session *sessions.Session, true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, + cropString(msg.Url), jsonString, ); err != nil { c.checkError("performance", err) @@ -655,6 +699,13 @@ func (c *connectorImpl) InsertRequest(session *sessions.Session, msg *messages.N true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, + cropString(msg.URL), nullableUint16(uint16(msg.Duration)), jsonString, ); err != nil { @@ -686,6 +737,13 @@ func (c *connectorImpl) InsertCustom(session *sessions.Session, msg *messages.Cu true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, + cropString(msg.Url), jsonString, ); err != nil { c.checkError("custom", err) @@ -717,6 +775,13 @@ func (c *connectorImpl) InsertGraphQL(session *sessions.Session, msg *messages.G true, session.Platform, session.UserOSVersion, + session.UserOS, + session.UserBrowser, + session.Referrer, + session.UserCountry, + session.UserState, + session.UserCity, + cropString(msg.Url), jsonString, ); err != nil { c.checkError("graphql", err)