jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(api): pages change base_path to path

Browse source 
feat(db): pages change base_path to path
This commit is contained in:
Taha Yassine Kraiem 2022-04-22 12:45:03 +02:00
parent d472e20948
commit 9891057105
7 changed files with 73 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api/chalicelib/core/events.py
View file

@ -339,7 +339,7 @@ def __generic_autocomplete(event: Event):
class event_type: class event_type:
CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label") CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label")
INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label") INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label")
LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="base_path") LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="path")
CUSTOM = Event(ui_type=schemas.EventType.custom, table="events_common.customs", column="name") CUSTOM = Event(ui_type=schemas.EventType.custom, table="events_common.customs", column="name")
REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="path") REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="path")
GRAPHQL = Event(ui_type=schemas.EventType.graphql, table="events.graphql", column="name") GRAPHQL = Event(ui_type=schemas.EventType.graphql, table="events.graphql", column="name")

2
api/chalicelib/core/insights.py
View file

@ -21,7 +21,7 @@ def __transform_journey(rows):
JOURNEY_DEPTH = 5 JOURNEY_DEPTH = 5
JOURNEY_TYPES = { JOURNEY_TYPES = {
"PAGES": {"table": "events.pages", "column": "base_path", "table_id": "message_id"}, "PAGES": {"table": "events.pages", "column": "path", "table_id": "message_id"},
"CLICK": {"table": "events.clicks", "column": "label", "table_id": "message_id"}, "CLICK": {"table": "events.clicks", "column": "label", "table_id": "message_id"},
# "VIEW": {"table": "events_ios.views", "column": "name", "table_id": "seq_index"}, TODO: enable this for SAAS only # "VIEW": {"table": "events_ios.views", "column": "name", "table_id": "seq_index"}, TODO: enable this for SAAS only
"EVENT": {"table": "events_common.customs", "column": "name", "table_id": "seq_index"} "EVENT": {"table": "events_common.customs", "column": "name", "table_id": "seq_index"}

4
api/chalicelib/core/sessions.py
View file

@ -353,8 +353,8 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d
full_args[arg_name] = metric_value[i] full_args[arg_name] = metric_value[i]
extra_where = f"WHERE ({' OR '.join(extra_where)})" extra_where = f"WHERE ({' OR '.join(extra_where)})"
elif metric_of == schemas.TableMetricOfType.visited_url: elif metric_of == schemas.TableMetricOfType.visited_url:
main_col = "base_path" main_col = "path"
extra_col = ", base_path" extra_col = ", path"
main_query = cur.mogrify(f"""{pre_query} main_query = cur.mogrify(f"""{pre_query}
SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values
FROM (SELECT {main_col} AS name, FROM (SELECT {main_col} AS name,

39
ee/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql
View file

@ -53,12 +53,36 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets
); );
ALTER TABLE events_common.requests ALTER TABLE events_common.requests
ADD COLUMN IF NOT EXISTS host text NULL, ADD COLUMN IF NOT EXISTS host text NULL,
ADD COLUMN IF NOT EXISTS base_path text NULL, ADD COLUMN IF NOT EXISTS path text NULL,
ADD COLUMN IF NOT EXISTS query text NULL; ADD COLUMN IF NOT EXISTS query text NULL;
ALTER TABLE events.pages ALTER TABLE events.pages
ADD COLUMN IF NOT EXISTS query text NULL; ADD COLUMN IF NOT EXISTS query text NULL;
DO
$$
BEGIN
IF EXISTS(SELECT *
FROM information_schema.columns
WHERE table_schema = 'events'
AND table_name = 'pages'
AND column_name = 'base_path')
THEN
ALTER TABLE events.pages
DROP COLUMN IF EXISTS path;
ALTER TABLE events.pages
RENAME COLUMN base_path TO path;
DROP INDEX IF EXISTS events.pages_base_path_gin_idx2;
DROP INDEX IF EXISTS pages_base_path_idx2;
ALTER INDEX IF EXISTS events.pages_base_path_gin_idx RENAME TO pages_path_gin_idx;
ALTER INDEX IF EXISTS events.pages_base_path_idx RENAME TO pages_path_idx;
ALTER INDEX IF EXISTS events.pages_base_path_session_id_timestamp_idx RENAME TO pages_path_session_id_timestamp_idx;
ALTER INDEX IF EXISTS events.pages_base_path_base_pathLNGT2_idx RENAME TO pages_path_pathLNGT2_idx;
END IF;
END
$$;
COMMIT; COMMIT;
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart';
@ -312,10 +336,13 @@ ON CONFLICT (predefined_key) DO UPDATE
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_path_nn_idx ON events_common.requests (path) WHERE path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_path_nn_gin_idx ON events_common.requests USING GIN (path gin_trgm_ops) WHERE path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp);
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2;

9
ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql
View file

@ -934,7 +934,6 @@ $$
timestamp bigint NOT NULL, timestamp bigint NOT NULL,
host text NOT NULL, host text NOT NULL,
path text NOT NULL, path text NOT NULL,
base_path text NOT NULL,
query text NULL, query text NULL,
referrer text DEFAULT NULL, referrer text DEFAULT NULL,
base_referrer text DEFAULT NULL, base_referrer text DEFAULT NULL,
@ -952,13 +951,9 @@ $$
PRIMARY KEY (session_id, message_id) PRIMARY KEY (session_id, message_id)
); );
CREATE INDEX IF NOT EXISTS pages_session_id_idx ON events.pages (session_id); CREATE INDEX IF NOT EXISTS pages_session_id_idx ON events.pages (session_id);
CREATE INDEX IF NOT EXISTS pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops);
CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops);
CREATE INDEX IF NOT EXISTS pages_timestamp_idx ON events.pages (timestamp); CREATE INDEX IF NOT EXISTS pages_timestamp_idx ON events.pages (timestamp);
CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (session_id, timestamp);
CREATE INDEX IF NOT EXISTS pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops);
CREATE INDEX IF NOT EXISTS pages_base_path_idx ON events.pages (base_path);
CREATE INDEX IF NOT EXISTS pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1));
CREATE INDEX IF NOT EXISTS pages_base_referrer_idx ON events.pages (base_referrer); CREATE INDEX IF NOT EXISTS pages_base_referrer_idx ON events.pages (base_referrer);
CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer, CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer,
length(base_referrer) - length(base_referrer) -
@ -993,8 +988,8 @@ $$
0; 0;
CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); CREATE INDEX IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp);
CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; CREATE INDEX IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2;
CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;

33
scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql
View file

@ -53,12 +53,36 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets
); );
ALTER TABLE events_common.requests ALTER TABLE events_common.requests
ADD COLUMN IF NOT EXISTS host text NULL, ADD COLUMN IF NOT EXISTS host text NULL,
ADD COLUMN IF NOT EXISTS path text NULL, ADD COLUMN IF NOT EXISTS path text NULL,
ADD COLUMN IF NOT EXISTS query text NULL; ADD COLUMN IF NOT EXISTS query text NULL;
ALTER TABLE events.pages ALTER TABLE events.pages
ADD COLUMN IF NOT EXISTS query text NULL; ADD COLUMN IF NOT EXISTS query text NULL;
DO
$$
BEGIN
IF EXISTS(SELECT *
FROM information_schema.columns
WHERE table_schema = 'events'
AND table_name = 'pages'
AND column_name = 'base_path')
THEN
ALTER TABLE events.pages
DROP COLUMN IF EXISTS path;
ALTER TABLE events.pages
RENAME COLUMN base_path TO path;
DROP INDEX IF EXISTS events.pages_base_path_gin_idx2;
DROP INDEX IF EXISTS pages_base_path_idx2;
ALTER INDEX IF EXISTS events.pages_base_path_gin_idx RENAME TO pages_path_gin_idx;
ALTER INDEX IF EXISTS events.pages_base_path_idx RENAME TO pages_path_idx;
ALTER INDEX IF EXISTS events.pages_base_path_session_id_timestamp_idx RENAME TO pages_path_session_id_timestamp_idx;
ALTER INDEX IF EXISTS events.pages_base_path_base_pathLNGT2_idx RENAME TO pages_path_pathLNGT2_idx;
END IF;
END
$$;
COMMIT; COMMIT;
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart';
@ -319,3 +343,6 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_comm
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp);
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2;

15
scripts/helm/db/init_dbs/postgresql/init_schema.sql
View file

@ -600,7 +600,6 @@ $$
-- --- events_common.sql --- -- --- events_common.sql ---
CREATE SCHEMA IF NOT EXISTS events_common;
CREATE TYPE events_common.custom_level AS ENUM ('info','error'); CREATE TYPE events_common.custom_level AS ENUM ('info','error');
@ -675,7 +674,6 @@ $$
CREATE INDEX requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE INDEX requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
-- --- events.sql --- -- --- events.sql ---
CREATE SCHEMA IF NOT EXISTS events;
CREATE TABLE events.pages CREATE TABLE events.pages
( (
@ -684,7 +682,6 @@ $$
timestamp bigint NOT NULL, timestamp bigint NOT NULL,
host text NOT NULL, host text NOT NULL,
path text NOT NULL, path text NOT NULL,
base_path text NOT NULL,
query text NULL, query text NULL,
referrer text DEFAULT NULL, referrer text DEFAULT NULL,
base_referrer text DEFAULT NULL, base_referrer text DEFAULT NULL,
@ -702,13 +699,9 @@ $$
PRIMARY KEY (session_id, message_id) PRIMARY KEY (session_id, message_id)
); );
CREATE INDEX pages_session_id_idx ON events.pages (session_id); CREATE INDEX pages_session_id_idx ON events.pages (session_id);
CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops);
CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops);
CREATE INDEX pages_timestamp_idx ON events.pages (timestamp); CREATE INDEX pages_timestamp_idx ON events.pages (timestamp);
CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp);
CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops);
CREATE INDEX pages_base_path_idx ON events.pages (base_path);
CREATE INDEX pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1));
CREATE INDEX pages_base_referrer_idx ON events.pages (base_referrer); CREATE INDEX pages_base_referrer_idx ON events.pages (base_referrer);
CREATE INDEX pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer, CREATE INDEX pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer,
length(base_referrer) - (CASE length(base_referrer) - (CASE
@ -739,10 +732,10 @@ $$
time_to_interactive > 0; time_to_interactive > 0;
CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL;
CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL;
CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); CREATE INDEX pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp);
CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; CREATE INDEX pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2;
CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; CREATE INDEX pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE INDEX pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE TABLE events.clicks CREATE TABLE events.clicks