diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index 81378a49c..272b86002 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -339,7 +339,7 @@ def __generic_autocomplete(event: Event): class event_type: CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label") INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label") - LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="base_path") + LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="path") CUSTOM = Event(ui_type=schemas.EventType.custom, table="events_common.customs", column="name") REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="path") GRAPHQL = Event(ui_type=schemas.EventType.graphql, table="events.graphql", column="name") diff --git a/api/chalicelib/core/insights.py b/api/chalicelib/core/insights.py index e5219344b..c04fd3981 100644 --- a/api/chalicelib/core/insights.py +++ b/api/chalicelib/core/insights.py @@ -21,7 +21,7 @@ def __transform_journey(rows): JOURNEY_DEPTH = 5 JOURNEY_TYPES = { - "PAGES": {"table": "events.pages", "column": "base_path", "table_id": "message_id"}, + "PAGES": {"table": "events.pages", "column": "path", "table_id": "message_id"}, "CLICK": {"table": "events.clicks", "column": "label", "table_id": "message_id"}, # "VIEW": {"table": "events_ios.views", "column": "name", "table_id": "seq_index"}, TODO: enable this for SAAS only "EVENT": {"table": "events_common.customs", "column": "name", "table_id": "seq_index"} diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 380ed2dd2..adc549d1e 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -353,8 +353,8 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d full_args[arg_name] = metric_value[i] extra_where = f"WHERE ({' OR '.join(extra_where)})" elif metric_of == schemas.TableMetricOfType.visited_url: - main_col = "base_path" - extra_col = ", base_path" + main_col = "path" + extra_col = ", path" main_query = cur.mogrify(f"""{pre_query} SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values FROM (SELECT {main_col} AS name, diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql index 7bb8e6b88..d7eeff911 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql @@ -53,12 +53,36 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets ); ALTER TABLE events_common.requests - ADD COLUMN IF NOT EXISTS host text NULL, - ADD COLUMN IF NOT EXISTS base_path text NULL, - ADD COLUMN IF NOT EXISTS query text NULL; + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS path text NULL, + ADD COLUMN IF NOT EXISTS query text NULL; ALTER TABLE events.pages ADD COLUMN IF NOT EXISTS query text NULL; + +DO +$$ + BEGIN + IF EXISTS(SELECT * + FROM information_schema.columns + WHERE table_schema = 'events' + AND table_name = 'pages' + AND column_name = 'base_path') + THEN + ALTER TABLE events.pages + DROP COLUMN IF EXISTS path; + ALTER TABLE events.pages + RENAME COLUMN base_path TO path; + DROP INDEX IF EXISTS events.pages_base_path_gin_idx2; + DROP INDEX IF EXISTS pages_base_path_idx2; + ALTER INDEX IF EXISTS events.pages_base_path_gin_idx RENAME TO pages_path_gin_idx; + ALTER INDEX IF EXISTS events.pages_base_path_idx RENAME TO pages_path_idx; + ALTER INDEX IF EXISTS events.pages_base_path_session_id_timestamp_idx RENAME TO pages_path_session_id_timestamp_idx; + ALTER INDEX IF EXISTS events.pages_base_path_base_pathLNGT2_idx RENAME TO pages_path_pathLNGT2_idx; + END IF; + END +$$; + COMMIT; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; @@ -312,10 +336,13 @@ ON CONFLICT (predefined_key) DO UPDATE CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_path_nn_idx ON events_common.requests (path) WHERE path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_path_nn_gin_idx ON events_common.requests USING GIN (path gin_trgm_ops) WHERE path IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; \ No newline at end of file +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp); +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 0b1692b20..461a414fc 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -934,7 +934,6 @@ $$ timestamp bigint NOT NULL, host text NOT NULL, path text NOT NULL, - base_path text NOT NULL, query text NULL, referrer text DEFAULT NULL, base_referrer text DEFAULT NULL, @@ -952,13 +951,9 @@ $$ PRIMARY KEY (session_id, message_id) ); CREATE INDEX IF NOT EXISTS pages_session_id_idx ON events.pages (session_id); - CREATE INDEX IF NOT EXISTS pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); CREATE INDEX IF NOT EXISTS pages_timestamp_idx ON events.pages (timestamp); CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); - CREATE INDEX IF NOT EXISTS pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); - CREATE INDEX IF NOT EXISTS pages_base_path_idx ON events.pages (base_path); - CREATE INDEX IF NOT EXISTS pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1)); CREATE INDEX IF NOT EXISTS pages_base_referrer_idx ON events.pages (base_referrer); CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer, length(base_referrer) - @@ -993,8 +988,8 @@ $$ 0; CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; - CREATE INDEX IF NOT EXISTS pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); - CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; + CREATE INDEX IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp); + CREATE INDEX IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2; CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; diff --git a/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql b/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql index 1c5e0a06f..d11cad5be 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql @@ -53,12 +53,36 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets ); ALTER TABLE events_common.requests - ADD COLUMN IF NOT EXISTS host text NULL, - ADD COLUMN IF NOT EXISTS path text NULL, - ADD COLUMN IF NOT EXISTS query text NULL; + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS path text NULL, + ADD COLUMN IF NOT EXISTS query text NULL; ALTER TABLE events.pages ADD COLUMN IF NOT EXISTS query text NULL; + +DO +$$ + BEGIN + IF EXISTS(SELECT * + FROM information_schema.columns + WHERE table_schema = 'events' + AND table_name = 'pages' + AND column_name = 'base_path') + THEN + ALTER TABLE events.pages + DROP COLUMN IF EXISTS path; + ALTER TABLE events.pages + RENAME COLUMN base_path TO path; + DROP INDEX IF EXISTS events.pages_base_path_gin_idx2; + DROP INDEX IF EXISTS pages_base_path_idx2; + ALTER INDEX IF EXISTS events.pages_base_path_gin_idx RENAME TO pages_path_gin_idx; + ALTER INDEX IF EXISTS events.pages_base_path_idx RENAME TO pages_path_idx; + ALTER INDEX IF EXISTS events.pages_base_path_session_id_timestamp_idx RENAME TO pages_path_session_id_timestamp_idx; + ALTER INDEX IF EXISTS events.pages_base_path_base_pathLNGT2_idx RENAME TO pages_path_pathLNGT2_idx; + END IF; + END +$$; + COMMIT; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; @@ -319,3 +343,6 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_comm CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp); +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index da2c7bc9c..5a01226f1 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -600,7 +600,6 @@ $$ -- --- events_common.sql --- - CREATE SCHEMA IF NOT EXISTS events_common; CREATE TYPE events_common.custom_level AS ENUM ('info','error'); @@ -675,7 +674,6 @@ $$ CREATE INDEX requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; -- --- events.sql --- - CREATE SCHEMA IF NOT EXISTS events; CREATE TABLE events.pages ( @@ -684,7 +682,6 @@ $$ timestamp bigint NOT NULL, host text NOT NULL, path text NOT NULL, - base_path text NOT NULL, query text NULL, referrer text DEFAULT NULL, base_referrer text DEFAULT NULL, @@ -702,13 +699,9 @@ $$ PRIMARY KEY (session_id, message_id) ); CREATE INDEX pages_session_id_idx ON events.pages (session_id); - CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); CREATE INDEX pages_timestamp_idx ON events.pages (timestamp); CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); - CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); - CREATE INDEX pages_base_path_idx ON events.pages (base_path); - CREATE INDEX pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1)); CREATE INDEX pages_base_referrer_idx ON events.pages (base_referrer); CREATE INDEX pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer, length(base_referrer) - (CASE @@ -739,10 +732,10 @@ $$ time_to_interactive > 0; CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; - CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); - CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; - CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; - CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + CREATE INDEX pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp); + CREATE INDEX pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2; + CREATE INDEX pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; + CREATE INDEX pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE TABLE events.clicks