From 7aafd8bc83d00a7c0b08a5ba8dd2b0e63a832b5a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 5 Jan 2022 19:21:31 +0100 Subject: [PATCH] feat(api): custom metrics CRUD feat(api): EE fixed add/edit project feat(api): refactored sessions search --- api/chalicelib/core/custom_metrics.py | 200 +++++ api/chalicelib/core/sessions.py | 1034 ++++++++++++------------- api/routers/core.py | 38 +- api/schemas.py | 30 + ee/api/chalicelib/core/projects.py | 13 +- 5 files changed, 788 insertions(+), 527 deletions(-) create mode 100644 api/chalicelib/core/custom_metrics.py diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py new file mode 100644 index 000000000..f716c4799 --- /dev/null +++ b/api/chalicelib/core/custom_metrics.py @@ -0,0 +1,200 @@ +import json + +import schemas +from chalicelib.utils import helper, pg_client +from chalicelib.utils.TimeUTC import TimeUTC + + +def try_live(project_id,data: schemas.CreateCustomMetricsSchema): + with pg_client.PostgresClient() as cur: + _data = {} + for i, s in enumerate(data.series): + for k in s.dict().keys(): + _data[f"{k}_{i}"] = s.__getattribute__(k) + _data[f"index_{i}"] = i + _data[f"filter_{i}"] = s.filter.json() + series_len = len(data.series) + data.series = None + params = { "project_id": project_id, **data.dict(), **_data} + query = cur.mogrify(f"""\ + WITH m AS (INSERT INTO metrics (project_id, user_id, title) + VALUES (%(project_id)s, %(user_id)s, %(title)s) + RETURNING *) + INSERT + INTO metric_series(metric_id, index, title, filter) + VALUES {",".join([f"((SELECT metric_id FROM m), %(index_{i})s, %(title_{i})s, %(filter_{i})s::jsonb)" + for i in range(series_len)])} + RETURNING metric_id;""", params) + + cur.execute( + query + ) + r = cur.fetchone() + r = helper.dict_to_camel_case(r) + return {"data": r} + +def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): + with pg_client.PostgresClient() as cur: + _data = {} + for i, s in enumerate(data.series): + for k in s.dict().keys(): + _data[f"{k}_{i}"] = s.__getattribute__(k) + _data[f"index_{i}"] = i + _data[f"filter_{i}"] = s.filter.json() + series_len = len(data.series) + data.series = None + params = {"user_id": user_id, "project_id": project_id, **data.dict(), **_data} + query = cur.mogrify(f"""\ + WITH m AS (INSERT INTO metrics (project_id, user_id, title) + VALUES (%(project_id)s, %(user_id)s, %(title)s) + RETURNING *) + INSERT + INTO metric_series(metric_id, index, title, filter) + VALUES {",".join([f"((SELECT metric_id FROM m), %(index_{i})s, %(title_{i})s, %(filter_{i})s::jsonb)" + for i in range(series_len)])} + RETURNING metric_id;""", params) + + cur.execute( + query + ) + r = cur.fetchone() + r = helper.dict_to_camel_case(r) + return {"data": r} + + +def __get_series_id(metric_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT series_id + FROM metric_series + WHERE metric_series.metric_id = %(metric_id)s + AND metric_series.deleted_at ISNULL;""", + {"metric_id": metric_id} + ) + ) + rows = cur.fetchall() + return [r["series_id"] for r in rows] + + +def update(metric_id, user_id, data: schemas.UpdateCustomMetricsSchema): + series_ids = __get_series_id(metric_id) + n_series = [] + d_series_ids = [] + u_series = [] + u_series_ids = [] + params = {"metric_id": metric_id, "is_public": data.is_public, "title": data.title} + for i, s in enumerate(data.series): + prefix = "u_" + if s.series_id is None: + n_series.append({"i": i, "s": s}) + prefix = "n_" + else: + u_series.append({"i": i, "s": s}) + u_series_ids.append(s.series_id) + ns = s.dict() + for k in ns.keys(): + if k == "filter": + ns[k] = json.dumps(ns[k]) + params[f"{prefix}{k}_{i}"] = ns[k] + for i in series_ids: + if i not in u_series_ids: + d_series_ids.append(i) + params["d_series_ids"] = tuple(d_series_ids) + + with pg_client.PostgresClient() as cur: + sub_queries = [] + if len(n_series) > 0: + sub_queries.append(f"""\ + n AS (INSERT INTO metric_series (metric_id, index, title, filter) + VALUES {",".join([f"(%(metric_id)s, %(n_index_{s['i']})s, %(n_title_{s['i']})s, %(n_filter_{s['i']})s::jsonb)" + for s in n_series])} + RETURNING 1)""") + if len(u_series) > 0: + sub_queries.append(f"""\ + u AS (UPDATE metric_series + SET title=series.title, + filter=series.filter, + index=series.filter.index + FROM (VALUES {",".join([f"(%(u_series_id_{s['i']})s,%(u_index_{s['i']})s,%(u_title_{s['i']})s,%(u_filter_{s['i']})s::jsonb)" + for s in n_series])}) AS series(series_id, index, title, filter) + WHERE metric_id =%(metric_id)s AND series_id=series.series_id + RETURNING 1)""") + if len(d_series_ids) > 0: + sub_queries.append("""\ + d AS (DELETE FROM metric_series WHERE metric_id =%(metric_id)s AND series_id IN %(d_series_ids)s + RETURNING 1)""") + query = cur.mogrify(f"""\ + {"WITH " if len(sub_queries) > 0 else ""}{",".join(sub_queries)} + UPDATE metrics + SET title = %(title)s, is_public= %(is_public)s WHERE metric_id = %(metric_id)s + RETURNING metric_id;""", params) + cur.execute( + query + ) + r = cur.fetchone() + r = helper.dict_to_camel_case(r) + return r + + +def get_all(project_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT * + FROM metrics + LEFT JOIN LATERAL (SELECT jsonb_agg(metric_series.* ORDER BY index) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE metrics.project_id = %(project_id)s + AND metrics.deleted_at ISNULL + AND (user_id = %(user_id)s OR is_public) + ORDER BY created_at;""", + {"project_id": project_id, "user_id": user_id} + ) + ) + rows = cur.fetchall() + for r in rows: + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + rows = helper.list_to_camel_case(rows) + return rows + + +def delete(project_id, metric_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.metrics + SET deleted_at = timezone('utc'::text, now()) + WHERE project_id = %(project_id)s + AND metric_id = %(metric_id)s;""", + {"metric_id": metric_id, "project_id": project_id, "user_id": user_id}) + ) + + return {"state": "success"} + + +def get(metric_id, project_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT * + FROM metrics + LEFT JOIN LATERAL (SELECT jsonb_agg(metric_series.* ORDER BY index) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE metrics.project_id = %(project_id)s + AND metrics.deleted_at ISNULL + AND (metrics.user_id = %(user_id)s OR metrics.is_public) + AND metrics.metric_id = %(metric_id)s + ORDER BY created_at;""", + {"metric_id": metric_id, "project_id": project_id, "user_id": user_id} + ) + ) + row = cur.fetchone() + row["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + return helper.dict_to_camel_case(row) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 8517b1161..9f4213add 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -162,519 +162,11 @@ def _isAny_opreator(op: schemas.SearchEventOperator): @dev.timed def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, favorite_only=False, errors_only=False, - error_status="ALL", - count_only=False, issue=None): + error_status="ALL", count_only=False, issue=None): + full_args, query_part, sort = search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, + user_id) + with pg_client.PostgresClient() as cur: - ss_constraints = [] - full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, - "projectId": project_id, "userId": user_id} - extra_constraints = [ - "s.project_id = %(project_id)s", - "s.duration IS NOT NULL" - ] - extra_from = "" - fav_only_join = "" - if favorite_only and not errors_only: - fav_only_join = "LEFT JOIN public.user_favorite_sessions AS fs ON fs.session_id = s.session_id" - extra_constraints.append("fs.user_id = %(userId)s") - full_args["userId"] = user_id - events_query_part = "" - - if len(data.filters) > 0: - meta_keys = None - for i, f in enumerate(data.filters): - if not isinstance(f.value, list): - f.value = [f.value] - if len(f.value) == 0 or f.value[0] is None: - continue - filter_type = f.type - # f.value = __get_sql_value_multiple(f.value) - f.value = helper.values_for_operator(value=f.value, op=f.operator) - f_k = f"f_value{i}" - full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} - op = __get_sql_operator(f.operator) \ - if filter_type not in [schemas.FilterType.events_count] else f.operator - is_any = _isAny_opreator(f.operator) - is_not = False - if __is_negation_operator(f.operator): - is_not = True - # op = __reverse_sql_operator(op) - if filter_type == schemas.FilterType.user_browser: - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.utm_source]: - if is_any: - extra_constraints.append('s.utm_source IS NOT NULL') - ss_constraints.append('ms.utm_source IS NOT NULL') - else: - extra_constraints.append( - _multiple_conditions(f's.utm_source {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.utm_source {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.utm_medium]: - if is_any: - extra_constraints.append('s.utm_medium IS NOT NULL') - ss_constraints.append('ms.utm_medium IS NOT NULL') - else: - extra_constraints.append( - _multiple_conditions(f's.utm_medium {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.utm_medium {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.utm_campaign]: - if is_any: - extra_constraints.append('s.utm_campaign IS NOT NULL') - ss_constraints.append('ms.utm_campaign IS NOT NULL') - else: - extra_constraints.append( - _multiple_conditions(f's.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - - elif filter_type == schemas.FilterType.duration: - if len(f.value) > 0 and f.value[0] is not None: - extra_constraints.append("s.duration >= %(minDuration)s") - ss_constraints.append("ms.duration >= %(minDuration)s") - full_args["minDuration"] = f.value[0] - if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: - extra_constraints.append("s.duration <= %(maxDuration)s") - ss_constraints.append("ms.duration <= %(maxDuration)s") - full_args["maxDuration"] = f.value[1] - elif filter_type == schemas.FilterType.referrer: - # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" - extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f"p.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type == events.event_type.METADATA.ui_type: - # get metadata list only if you need it - if meta_keys is None: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} - # op = __get_sql_operator(f.operator) - if f.key in meta_keys.keys(): - extra_constraints.append( - _multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", - f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", - f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: - # op = __get_sql_operator(f.operator) - extra_constraints.append( - _multiple_conditions(f"s.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [schemas.FilterType.user_anonymous_id, - schemas.FilterType.user_anonymous_id_ios]: - # op = __get_sql_operator(f.operator) - extra_constraints.append( - _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: - # op = __get_sql_operator(f.operator) - extra_constraints.append( - _multiple_conditions(f"s.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type == schemas.FilterType.platform: - # op = __get_sql_operator(f.operator) - extra_constraints.append( - _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type == schemas.FilterType.issue: - extra_constraints.append( - _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type == schemas.FilterType.events_count: - extra_constraints.append( - _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - - # --------------------------------------------------------------------------- - if len(data.events) > 0: - # ss_constraints = [s.decode('UTF-8') for s in ss_constraints] - events_query_from = [] - event_index = 0 - or_events = data.events_order == schemas.SearchEventOrder._or - # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " - events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " - for i, event in enumerate(data.events): - event_type = event.type - is_any = _isAny_opreator(event.operator) - if not isinstance(event.value, list): - event.value = [event.value] - op = __get_sql_operator(event.operator) - is_not = False - if __is_negation_operator(event.operator): - is_not = True - op = __reverse_sql_operator(op) - if event_index == 0 or or_events: - event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" - event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", - "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", - "ms.start_ts <= %(endDate)s", "ms.duration IS NOT NULL"] - else: - event_from = "%s" - event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", - "main.session_id=event_0.session_id"] - if data.events_order == schemas.SearchEventOrder._then: - event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") - e_k = f"e_value{i}" - if event.type != schemas.PerformanceEventType.time_between_events: - event.value = helper.values_for_operator(value=event.value, op=event.operator) - full_args = {**full_args, **_multiple_values(event.value, value_key=e_k)} - - # if event_type not in list(events.SUPPORTED_TYPES.keys()) \ - # or event.value in [None, "", "*"] \ - # and (event_type != events.event_type.ERROR.ui_type \ - # or event_type != events.event_type.ERROR_IOS.ui_type): - # continue - if event_type == events.event_type.CLICK.ui_type: - event_from = event_from % f"{events.event_type.CLICK.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, - value_key=e_k)) - - elif event_type == events.event_type.INPUT.ui_type: - event_from = event_from % f"{events.event_type.INPUT.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, - value_key=e_k)) - if event.custom is not None and len(event.custom) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, - value_key=f"custom{i}")) - full_args = {**full_args, **_multiple_values(event.custom, value_key=f"custom{i}")} - - elif event_type == events.event_type.LOCATION.ui_type: - event_from = event_from % f"{events.event_type.LOCATION.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.CUSTOM.ui_type: - event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, - value_key=e_k)) - elif event_type == events.event_type.REQUEST.ui_type: - event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, - value_key=e_k)) - elif event_type == events.event_type.GRAPHQL.ui_type: - event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, - value_key=e_k)) - elif event_type == events.event_type.STATEACTION.ui_type: - event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.ERROR.ui_type: - # if event.source in [None, "*", ""]: - # event.source = "js_exception" - event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" - if event.value not in [None, "*", ""]: - if not is_any: - event_where.append(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)") - if event.source not in [None, "*", ""]: - event_where.append(f"main1.source = %(source)s") - full_args["source"] = event.source - elif event.source not in [None, "*", ""]: - event_where.append(f"main1.source = %(source)s") - full_args["source"] = event.source - - # ----- IOS - elif event_type == events.event_type.CLICK_IOS.ui_type: - event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - - elif event_type == events.event_type.INPUT_IOS.ui_type: - event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - if event.custom is not None and len(event.custom) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, - value_key="custom{i}")) - full_args = {**full_args, **_multiple_values(event.custom, f"custom{i}")} - elif event_type == events.event_type.VIEW_IOS.ui_type: - event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.CUSTOM_IOS.ui_type: - event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.REQUEST_IOS.ui_type: - event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.ERROR_IOS.ui_type: - event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" - if not is_any and event.value not in [None, "*", ""]: - event_where.append( - _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", - event.value, value_key=e_k)) - elif event_type == schemas.PerformanceEventType.fetch_failed: - event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", - event.value, value_key=e_k)) - col = performance_event.get_col(event_type) - colname = col["column"] - event_where.append(f"main.{colname} = FALSE") - # elif event_type == schemas.PerformanceEventType.fetch_duration: - # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - # if not is_any: - # event_where.append( - # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", - # event.value, value_key=e_k)) - # col = performance_event.get_col(event_type) - # colname = col["column"] - # tname = "main" - # e_k += "_custom" - # full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} - # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - # _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", - # event.custom, value_key=e_k)) - elif event_type in [schemas.PerformanceEventType.location_dom_complete, - schemas.PerformanceEventType.location_largest_contentful_paint_time, - schemas.PerformanceEventType.location_ttfb, - schemas.PerformanceEventType.location_avg_cpu_load, - schemas.PerformanceEventType.location_avg_memory_usage - ]: - event_from = event_from % f"{events.event_type.LOCATION.table} AS main " - col = performance_event.get_col(event_type) - colname = col["column"] - tname = "main" - if col.get("extraJoin") is not None: - tname = "ej" - event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" - event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", - f"{tname}.timestamp <= %(endDate)s"] - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} - - event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", - event.custom, value_key=e_k)) - elif event_type == schemas.PerformanceEventType.time_between_events: - event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " - if not isinstance(event.value[0].value, list): - event.value[0].value = [event.value[0].value] - if not isinstance(event.value[1].value, list): - event.value[1].value = [event.value[1].value] - event.value[0].value = helper.values_for_operator(value=event.value[0].value, - op=event.value[0].operator) - event.value[1].value = helper.values_for_operator(value=event.value[1].value, - op=event.value[0].operator) - e_k1 = e_k + "_e1" - e_k2 = e_k + "_e2" - full_args = {**full_args, - **_multiple_values(event.value[0].value, value_key=e_k1), - **_multiple_values(event.value[1].value, value_key=e_k2)} - s_op = __get_sql_operator(event.value[0].operator) - event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] - if event_index > 0 and not or_events: - event_where.append("main2.session_id=event_0.session_id") - event_where.append( - _multiple_conditions( - f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", - event.value[0].value, value_key=e_k1)) - s_op = __get_sql_operator(event.value[1].operator) - event_where.append( - _multiple_conditions( - f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", - event.value[1].value, value_key=e_k2)) - - e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} - event_where.append( - _multiple_conditions(f"main2.timestamp - main.timestamp {event.customOperator} %({e_k})s", - event.custom, value_key=e_k)) - - - else: - continue - if event_index == 0 or or_events: - event_where += ss_constraints - if is_not: - if event_index == 0 or or_events: - events_query_from.append(f"""\ - (SELECT - session_id, - 0 AS timestamp - FROM sessions - WHERE EXISTS(SELECT session_id - FROM {event_from} - WHERE {" AND ".join(event_where)} - AND sessions.session_id=ms.session_id) IS FALSE - AND project_id = %(projectId)s - AND start_ts >= %(startDate)s - AND start_ts <= %(endDate)s - AND duration IS NOT NULL - ) {"" if or_events else ("AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ - """) - else: - events_query_from.append(f"""\ - (SELECT - event_0.session_id, - event_{event_index - 1}.timestamp AS timestamp - WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE - ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ - """) - else: - events_query_from.append(f"""\ - (SELECT main.session_id, MIN(main.timestamp) AS timestamp - FROM {event_from} - WHERE {" AND ".join(event_where)} - GROUP BY 1 - ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ - """) - event_index += 1 - if event_index > 0: - if or_events: - events_query_part = f"""SELECT - session_id, - MIN(timestamp) AS first_event_ts, - MAX(timestamp) AS last_event_ts - FROM ({events_joiner.join(events_query_from)}) AS u - GROUP BY 1 - {fav_only_join}""" - else: - events_query_part = f"""SELECT - event_0.session_id, - MIN(event_0.timestamp) AS first_event_ts, - MAX(event_{event_index - 1}.timestamp) AS last_event_ts - FROM {events_joiner.join(events_query_from)} - GROUP BY 1 - {fav_only_join}""" - else: - data.events = [] - - # --------------------------------------------------------------------------- - - if data.startDate is not None: - extra_constraints.append("s.start_ts >= %(startDate)s") - - if data.endDate is not None: - extra_constraints.append("s.start_ts <= %(endDate)s") - - # if data.platform is not None: - # if data.platform == schemas.PlatformType.mobile: - # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") - # elif data.platform == schemas.PlatformType.desktop: - # extra_constraints.append( - # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") - - order = "DESC" - if data.order is not None: - order = data.order - sort = 'session_id' - if data.sort is not None and data.sort != "session_id": - sort += " " + order + "," + helper.key_to_snake_case(data.sort) - else: - sort = 'session_id' - - if errors_only: - extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" - extra_constraints.append(b"ser.source = 'js_exception'") - if error_status != "ALL": - extra_constraints.append(cur.mogrify("ser.status = %(status)s", {"status": error_status.lower()})) - if favorite_only: - extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" - extra_constraints.append(cur.mogrify("ufe.user_id = %(user_id)s", {"user_id": user_id})) - - # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] - if not favorite_only and not errors_only: - extra_from += """LEFT JOIN (SELECT user_id, session_id - FROM public.user_favorite_sessions - WHERE user_id = %(userId)s) AS favorite_sessions - USING (session_id)""" - extra_join = "" - if issue is not None: - extra_join = cur.mogrify(""" - INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) - WHERE issues.session_id=f.session_id - AND p_issues.type=%(type)s - AND p_issues.context_string=%(contextString)s - AND timestamp >= f.first_event_ts - AND timestamp <= f.last_event_ts) AS issues ON(TRUE) - """, {"contextString": issue["contextString"], "type": issue["type"]}).decode('UTF-8') - - query_part = f"""\ - FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} - {extra_join} - {"INNER JOIN public.sessions AS s USING(session_id)" if len(events_query_part) > 0 else ""} - {extra_from} - WHERE - - {" AND ".join(extra_constraints)}""" - if errors_only: main_query = cur.mogrify(f"""SELECT DISTINCT er.error_id, ser.status, ser.parent_error_id, ser.payload, COALESCE((SELECT TRUE @@ -685,21 +177,19 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f FROM public.user_viewed_errors AS ve WHERE er.error_id = ve.error_id AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed - {query_part};""", - full_args) + {query_part};""", full_args) elif count_only: - main_query = cur.mogrify( - f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, COUNT(DISTINCT s.user_uuid) AS count_users - {query_part};""", - full_args) + main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, + COUNT(DISTINCT s.user_uuid) AS count_users + {query_part};""", full_args) else: main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) FILTER (WHERE rn <= 200), '[]'::JSONB) AS sessions FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY favorite DESC, issue_score DESC, session_id desc, start_ts desc) AS rn FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} {query_part} ORDER BY s.session_id desc) AS filtred_sessions - ORDER BY favorite DESC, issue_score DESC, {sort} {order}) AS full_sessions;""", + ORDER BY favorite DESC, issue_score DESC, {sort} {data.order}) AS full_sessions;""", full_args) # main_query = cur.mogrify(f"""SELECT * FROM @@ -713,7 +203,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f print(main_query) cur.execute(main_query) - print("Executed--------------------") + print("--------------------") if count_only: return helper.dict_to_camel_case(cur.fetchone()) sessions = cur.fetchone() @@ -740,6 +230,510 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f } +def search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, user_id): + ss_constraints = [] + full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, + "projectId": project_id, "userId": user_id} + extra_constraints = [ + "s.project_id = %(project_id)s", + "s.duration IS NOT NULL" + ] + extra_from = "" + fav_only_join = "" + if favorite_only and not errors_only: + fav_only_join = "LEFT JOIN public.user_favorite_sessions AS fs ON fs.session_id = s.session_id" + extra_constraints.append("fs.user_id = %(userId)s") + full_args["userId"] = user_id + events_query_part = "" + if len(data.filters) > 0: + meta_keys = None + for i, f in enumerate(data.filters): + if not isinstance(f.value, list): + f.value = [f.value] + if len(f.value) == 0 or f.value[0] is None: + continue + filter_type = f.type + # f.value = __get_sql_value_multiple(f.value) + f.value = helper.values_for_operator(value=f.value, op=f.operator) + f_k = f"f_value{i}" + full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} + op = __get_sql_operator(f.operator) \ + if filter_type not in [schemas.FilterType.events_count] else f.operator + is_any = _isAny_opreator(f.operator) + is_not = False + if __is_negation_operator(f.operator): + is_not = True + # op = __reverse_sql_operator(op) + if filter_type == schemas.FilterType.user_browser: + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.utm_source]: + if is_any: + extra_constraints.append('s.utm_source IS NOT NULL') + ss_constraints.append('ms.utm_source IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_source {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_source {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_medium]: + if is_any: + extra_constraints.append('s.utm_medium IS NOT NULL') + ss_constraints.append('ms.utm_medium IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_medium {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_medium {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_campaign]: + if is_any: + extra_constraints.append('s.utm_campaign IS NOT NULL') + ss_constraints.append('ms.utm_campaign IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + + elif filter_type == schemas.FilterType.duration: + if len(f.value) > 0 and f.value[0] is not None: + extra_constraints.append("s.duration >= %(minDuration)s") + ss_constraints.append("ms.duration >= %(minDuration)s") + full_args["minDuration"] = f.value[0] + if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: + extra_constraints.append("s.duration <= %(maxDuration)s") + ss_constraints.append("ms.duration <= %(maxDuration)s") + full_args["maxDuration"] = f.value[1] + elif filter_type == schemas.FilterType.referrer: + # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" + extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f"p.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == events.event_type.METADATA.ui_type: + # get metadata list only if you need it + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + # op = __get_sql_operator(f.operator) + if f.key in meta_keys.keys(): + extra_constraints.append( + _multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", + f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", + f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == schemas.FilterType.platform: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.issue: + extra_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.events_count: + extra_constraints.append( + _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + # --------------------------------------------------------------------------- + if len(data.events) > 0: + # ss_constraints = [s.decode('UTF-8') for s in ss_constraints] + events_query_from = [] + event_index = 0 + or_events = data.events_order == schemas.SearchEventOrder._or + # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " + events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " + for i, event in enumerate(data.events): + event_type = event.type + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + op = __get_sql_operator(event.operator) + is_not = False + if __is_negation_operator(event.operator): + is_not = True + op = __reverse_sql_operator(op) + if event_index == 0 or or_events: + event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" + event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", + "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", + "ms.start_ts <= %(endDate)s", "ms.duration IS NOT NULL"] + else: + event_from = "%s" + event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", + "main.session_id=event_0.session_id"] + if data.events_order == schemas.SearchEventOrder._then: + event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") + e_k = f"e_value{i}" + if event.type != schemas.PerformanceEventType.time_between_events: + event.value = helper.values_for_operator(value=event.value, op=event.operator) + full_args = {**full_args, **_multiple_values(event.value, value_key=e_k)} + + # if event_type not in list(events.SUPPORTED_TYPES.keys()) \ + # or event.value in [None, "", "*"] \ + # and (event_type != events.event_type.ERROR.ui_type \ + # or event_type != events.event_type.ERROR_IOS.ui_type): + # continue + if event_type == events.event_type.CLICK.ui_type: + event_from = event_from % f"{events.event_type.CLICK.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) + + elif event_type == events.event_type.INPUT.ui_type: + event_from = event_from % f"{events.event_type.INPUT.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) + if event.custom is not None and len(event.custom) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, + value_key=f"custom{i}")) + full_args = {**full_args, **_multiple_values(event.custom, value_key=f"custom{i}")} + + elif event_type == events.event_type.LOCATION.ui_type: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.CUSTOM.ui_type: + event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.REQUEST.ui_type: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.GRAPHQL.ui_type: + event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.STATEACTION.ui_type: + event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR.ui_type: + # if event.source in [None, "*", ""]: + # event.source = "js_exception" + event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" + if event.value not in [None, "*", ""]: + if not is_any: + event_where.append(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)") + if event.source not in [None, "*", ""]: + event_where.append(f"main1.source = %(source)s") + full_args["source"] = event.source + elif event.source not in [None, "*", ""]: + event_where.append(f"main1.source = %(source)s") + full_args["source"] = event.source + + # ----- IOS + elif event_type == events.event_type.CLICK_IOS.ui_type: + event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + + elif event_type == events.event_type.INPUT_IOS.ui_type: + event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + if event.custom is not None and len(event.custom) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, + value_key="custom{i}")) + full_args = {**full_args, **_multiple_values(event.custom, f"custom{i}")} + elif event_type == events.event_type.VIEW_IOS.ui_type: + event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.CUSTOM_IOS.ui_type: + event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.REQUEST_IOS.ui_type: + event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR_IOS.ui_type: + event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.fetch_failed: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) + col = performance_event.get_col(event_type) + colname = col["column"] + event_where.append(f"main.{colname} = FALSE") + # elif event_type == schemas.PerformanceEventType.fetch_duration: + # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + # if not is_any: + # event_where.append( + # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + # event.value, value_key=e_k)) + # col = performance_event.get_col(event_type) + # colname = col["column"] + # tname = "main" + # e_k += "_custom" + # full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + # _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", + # event.custom, value_key=e_k)) + elif event_type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ]: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + col = performance_event.get_col(event_type) + colname = col["column"] + tname = "main" + if col.get("extraJoin") is not None: + tname = "ej" + event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" + event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", + f"{tname}.timestamp <= %(endDate)s"] + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + + event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", + event.custom, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.time_between_events: + event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " + if not isinstance(event.value[0].value, list): + event.value[0].value = [event.value[0].value] + if not isinstance(event.value[1].value, list): + event.value[1].value = [event.value[1].value] + event.value[0].value = helper.values_for_operator(value=event.value[0].value, + op=event.value[0].operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, + op=event.value[0].operator) + e_k1 = e_k + "_e1" + e_k2 = e_k + "_e2" + full_args = {**full_args, + **_multiple_values(event.value[0].value, value_key=e_k1), + **_multiple_values(event.value[1].value, value_key=e_k2)} + s_op = __get_sql_operator(event.value[0].operator) + event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] + if event_index > 0 and not or_events: + event_where.append("main2.session_id=event_0.session_id") + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) + s_op = __get_sql_operator(event.value[1].operator) + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) + + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + event_where.append( + _multiple_conditions(f"main2.timestamp - main.timestamp {event.customOperator} %({e_k})s", + event.custom, value_key=e_k)) + + + else: + continue + if event_index == 0 or or_events: + event_where += ss_constraints + if is_not: + if event_index == 0 or or_events: + events_query_from.append(f"""\ + (SELECT + session_id, + 0 AS timestamp + FROM sessions + WHERE EXISTS(SELECT session_id + FROM {event_from} + WHERE {" AND ".join(event_where)} + AND sessions.session_id=ms.session_id) IS FALSE + AND project_id = %(projectId)s + AND start_ts >= %(startDate)s + AND start_ts <= %(endDate)s + AND duration IS NOT NULL + ) {"" if or_events else ("AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + else: + events_query_from.append(f"""\ + (SELECT + event_0.session_id, + event_{event_index - 1}.timestamp AS timestamp + WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE + ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ + """) + else: + events_query_from.append(f"""\ + (SELECT main.session_id, MIN(main.timestamp) AS timestamp + FROM {event_from} + WHERE {" AND ".join(event_where)} + GROUP BY 1 + ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + event_index += 1 + if event_index > 0: + if or_events: + events_query_part = f"""SELECT + session_id, + MIN(timestamp) AS first_event_ts, + MAX(timestamp) AS last_event_ts + FROM ({events_joiner.join(events_query_from)}) AS u + GROUP BY 1 + {fav_only_join}""" + else: + events_query_part = f"""SELECT + event_0.session_id, + MIN(event_0.timestamp) AS first_event_ts, + MAX(event_{event_index - 1}.timestamp) AS last_event_ts + FROM {events_joiner.join(events_query_from)} + GROUP BY 1 + {fav_only_join}""" + else: + data.events = [] + # --------------------------------------------------------------------------- + if data.startDate is not None: + extra_constraints.append("s.start_ts >= %(startDate)s") + if data.endDate is not None: + extra_constraints.append("s.start_ts <= %(endDate)s") + # if data.platform is not None: + # if data.platform == schemas.PlatformType.mobile: + # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + # elif data.platform == schemas.PlatformType.desktop: + # extra_constraints.append( + # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + if data.order is None: + data.order = "DESC" + sort = 'session_id' + if data.sort is not None and data.sort != "session_id": + sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) + else: + sort = 'session_id' + if errors_only: + extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" + extra_constraints.append("ser.source = 'js_exception'") + if error_status != "ALL": + extra_constraints.append("ser.status = %(error_status)s") + full_args["status"] = error_status.lower() + if favorite_only: + extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" + extra_constraints.append("ufe.user_id = %(user_id)s") + # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] + if not favorite_only and not errors_only: + extra_from += """LEFT JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id)""" + extra_join = "" + if issue is not None: + extra_join = """ + INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) + WHERE issues.session_id=f.session_id + AND p_issues.type=%(issue_type)s + AND p_issues.context_string=%(issue_contextString)s + AND timestamp >= f.first_event_ts + AND timestamp <= f.last_event_ts) AS issues ON(TRUE) + """ + full_args["issue_contextString"] = issue["contextString"] + full_args["issue_type"] = issue["type"] + query_part = f"""\ + FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} + {extra_join} + {"INNER JOIN public.sessions AS s USING(session_id)" if len(events_query_part) > 0 else ""} + {extra_from} + WHERE + {" AND ".join(extra_constraints)}""" + return full_args, query_part, sort + + def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): if project_id is None: all_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) diff --git a/api/routers/core.py b/api/routers/core.py index a509db750..df80a110a 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -10,7 +10,8 @@ from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assig log_tool_stackdriver, reset_password, sessions_favorite_viewed, \ log_tool_cloudwatch, log_tool_sentry, log_tool_sumologic, log_tools, errors, sessions, \ log_tool_newrelic, announcements, log_tool_bugsnag, weekly_report, integration_jira_cloud, integration_github, \ - assist, heatmaps, mobile, signup, tenants, errors_favorite_viewed, boarding, notifications, webhook, slack, users + assist, heatmaps, mobile, signup, tenants, errors_favorite_viewed, boarding, notifications, webhook, slack, users, \ + custom_metrics from chalicelib.core.collaboration_slack import Slack from chalicelib.utils import email_helper from chalicelib.utils.TimeUTC import TimeUTC @@ -1086,3 +1087,38 @@ def change_client_password(data: schemas.EditUserPasswordSchema = Body(...), return users.change_password(email=context.email, old_password=data.old_password, new_password=data.new_password, tenant_id=context.tenant_id, user_id=context.user_id) + + +@app.post('/{projectId}/custom_metrics/try', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) +def try_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return custom_metrics.try_live(project_id=projectId, data=data) + +@app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) +def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return custom_metrics.create(project_id=projectId, user_id=context.user_id, data=data) + + +@app.get('/{projectId}/custom_metrics', tags=["customMetrics"]) +def get_custom_metrics(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.get_all(project_id=projectId, user_id=context.user_id)} + + +@app.get('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.get(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} + + +@app.post('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.update(user_id=context.user_id, metric_id=metric_id,data=data)} + + +@app.delete('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def delete_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.delete(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} diff --git a/api/schemas.py b/api/schemas.py index fe5296e52..edbcdbdcb 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -576,3 +576,33 @@ class SentrySchema(BaseModel): class MobileSignPayloadSchema(BaseModel): keys: List[str] = Field(...) + + +class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): + startDate: Optional[int] = Field(None) + endDate: Optional[int] = Field(None) + sort: Optional[str] = Field(None) + order: Optional[str] = Field(None) + + +class CustomMetricCreateSeriesSchema(BaseModel): + title: Optional[str] = Field(None) + index: Optional[int] = Field(None) + filter: Optional[CustomMetricSeriesFilterSchema] = Field([]) + + +class CreateCustomMetricsSchema(BaseModel): + title: str = Field(...) + series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) + is_public: Optional[bool] = Field(False) + + class Config: + alias_generator = attribute_to_camel_case + + +class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): + series_id: Optional[int] = Field(None) + + +class UpdateCustomMetricsSchema(CreateCustomMetricsSchema): + series: List[CustomMetricUpdateSeriesSchema] = Field(..., min_items=1) diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index a9d7767d0..75efdd452 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -1,5 +1,6 @@ import json +import schemas from chalicelib.core import users from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC @@ -104,8 +105,8 @@ def get_project(tenant_id, project_id, include_last_session=False, include_gdpr= query = cur.mogrify(f"""\ SELECT s.project_id, - s.name, - s.project_key + s.project_key, + s.name {",(SELECT max(ss.start_ts) FROM public.sessions AS ss WHERE ss.project_id = %(project_id)s) AS last_recorded_session_at" if include_last_session else ""} {',s.gdpr' if include_gdpr else ''} {tracker_query} @@ -129,20 +130,20 @@ def is_authorized(project_id, tenant_id): return get_project(tenant_id=tenant_id, project_id=project_id) is not None -def create(tenant_id, user_id, data, skip_authorization=False): +def create(tenant_id, user_id, data: schemas.CreateProjectSchema, skip_authorization=False): if not skip_authorization: admin = users.get(user_id=user_id, tenant_id=tenant_id) if not admin["admin"] and not admin["superAdmin"]: return {"errors": ["unauthorized"]} - return {"data": __create(tenant_id=tenant_id, name=data.get("name", "my first project"))} + return {"data": __create(tenant_id=tenant_id, name=data.name)} -def edit(tenant_id, user_id, project_id, data): +def edit(tenant_id, user_id, project_id, data: schemas.CreateProjectSchema): admin = users.get(user_id=user_id, tenant_id=tenant_id) if not admin["admin"] and not admin["superAdmin"]: return {"errors": ["unauthorized"]} return {"data": __update(tenant_id=tenant_id, project_id=project_id, - changes={"name": data.get("name", "my first project")})} + changes={"name": data.name})} def delete(tenant_id, user_id, project_id):