From 58de043193770a00035050a4100e796abbbc53f2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 28 Dec 2022 19:45:47 +0100 Subject: [PATCH] feat(chalice): search sessions by click-rage-selector feat(chalice): support empty cards feat(DB): remove unused indexes --- api/chalicelib/core/custom_metrics.py | 7 +- api/chalicelib/core/events.py | 2 +- api/chalicelib/core/funnels.py | 3 +- api/chalicelib/core/metadata.py | 10 +- api/chalicelib/core/sessions.py | 358 ++++++++---------- api/chalicelib/core/sessions_notes.py | 5 +- api/chalicelib/core/significance.py | 41 +- api/schemas.py | 33 +- ee/api/.gitignore | 1 + ee/api/chalicelib/core/autocomplete_exp.py | 2 +- ee/api/chalicelib/core/sessions.py | 329 +++++++--------- ee/api/clean.sh | 1 + .../db/init_dbs/postgresql/1.9.5/1.9.5.sql | 16 + .../db/init_dbs/postgresql/1.9.5/1.9.5.sql | 17 + 14 files changed, 395 insertions(+), 430 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 01067cac2..3c6e6fd68 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -125,7 +125,8 @@ def merged_live(project_id, data: schemas.CreateCardSchema, user_id=None): return __get_sessions_list(project_id=project_id, user_id=user_id, data=data) elif __is_click_map(data): return __get_click_map_chat(project_id=project_id, user_id=user_id, data=data) - + elif len(data.series) == 0: + return [] series_charts = __try_live(project_id=project_id, data=data) if data.view_type == schemas.MetricTimeseriesViewType.progress or data.metric_type == schemas.MetricType.table: return series_charts @@ -376,7 +377,9 @@ def search_all(project_id, user_id, data: schemas.SearchCardsSchema, include_ser AND metric_series.deleted_at ISNULL ) AS metric_series ON (TRUE)""" query = cur.mogrify( - f"""SELECT * + f"""SELECT metric_id, project_id, user_id, name, is_public, created_at, edited_at, + metric_type, metric_of, metric_format, metric_value, view_type, is_pinned, + predefined_key, dashboards, owner_email, default_config AS config, thumbnail FROM metrics {sub_join} LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index 1662fceee..abed5d8cf 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -42,7 +42,7 @@ def __get_grouped_clickrage(rows, session_id, project_id): for c in click_rage_issues: merge_count = c.get("payload") if merge_count is not None: - merge_count = merge_count.get("count", 3) + merge_count = merge_count.get("Count", 3) else: merge_count = 3 for i in range(len(rows)): diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 65cb7b09a..c77b19a19 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -2,6 +2,7 @@ import json from typing import List import chalicelib.utils.helper +import chalicelib.utils.sql_helper import schemas from chalicelib.core import significance, sessions from chalicelib.utils import dev @@ -39,7 +40,7 @@ def __fix_stages(f_events: List[schemas._SessionSearchEventSchema]): if not isinstance(e.value, list): e.value = [e.value] - is_any = sessions._isAny_opreator(e.operator) + is_any = chalicelib.utils.sessions_helper._isAny_opreator(e.operator) if not is_any and isinstance(e.value, list) and len(e.value) == 0: continue events.append(e) diff --git a/api/chalicelib/core/metadata.py b/api/chalicelib/core/metadata.py index 0f73d0962..4c6d3a580 100644 --- a/api/chalicelib/core/metadata.py +++ b/api/chalicelib/core/metadata.py @@ -6,7 +6,7 @@ from chalicelib.utils import pg_client MAX_INDEXES = 10 -def _get_column_names(): +def column_names(): return [f"metadata_{i}" for i in range(1, MAX_INDEXES + 1)] @@ -16,7 +16,7 @@ def get(project_id): cur.mogrify( f"""\ SELECT - {",".join(_get_column_names())} + {",".join(column_names())} FROM public.projects WHERE project_id = %(project_id)s AND deleted_at ISNULL LIMIT 1;""", {"project_id": project_id}) @@ -38,7 +38,7 @@ def get_batch(project_ids): cur.mogrify( f"""\ SELECT - project_id, {",".join(_get_column_names())} + project_id, {",".join(column_names())} FROM public.projects WHERE project_id IN %(project_ids)s AND deleted_at ISNULL;""", {"project_ids": tuple(project_ids)}) @@ -140,7 +140,7 @@ def add(tenant_id, project_id, new_name): def search(tenant_id, project_id, key, value): value = value + "%" s_query = [] - for f in _get_column_names(): + for f in column_names(): s_query.append(f"CASE WHEN {f}=%(key)s THEN TRUE ELSE FALSE END AS {f}") with pg_client.PostgresClient() as cur: @@ -215,7 +215,7 @@ def get_keys_by_projects(project_ids): f"""\ SELECT project_id, - {",".join(_get_column_names())} + {",".join(column_names())} FROM public.projects WHERE project_id IN %(project_ids)s AND deleted_at ISNULL;""", {"project_ids": tuple(project_ids)}) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 5e176009c..4618b2918 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1,10 +1,11 @@ -from typing import List, Union +from typing import List import schemas from chalicelib.core import events, metadata, events_ios, \ - sessions_mobs, issues, projects, errors, resources, assist, performance_event, sessions_viewed, sessions_favorite, \ + sessions_mobs, issues, projects, errors, resources, assist, performance_event, sessions_favorite, \ sessions_devtool, sessions_notes from chalicelib.utils import pg_client, helper, metrics_helper +from chalicelib.utils import sql_helper as sh SESSION_PROJECTION_COLS = """s.project_id, s.session_id::text AS session_id, @@ -60,7 +61,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_ s.session_id::text AS session_id, (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} - {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata._get_column_names()]) + ") AS project_metadata") if group_metadata else ''} + {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''} FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} WHERE s.project_id = %(project_id)s AND s.session_id = %(session_id)s;""", @@ -114,67 +115,6 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_ return None -def __get_sql_operator(op: Union[schemas.SearchEventOperator, schemas.ClickEventExtraOperator]): - return { - schemas.SearchEventOperator._is: "=", - schemas.SearchEventOperator._is_any: "IN", - schemas.SearchEventOperator._on: "=", - schemas.SearchEventOperator._on_any: "IN", - schemas.SearchEventOperator._is_not: "!=", - schemas.SearchEventOperator._not_on: "!=", - schemas.SearchEventOperator._contains: "ILIKE", - schemas.SearchEventOperator._not_contains: "NOT ILIKE", - schemas.SearchEventOperator._starts_with: "ILIKE", - schemas.SearchEventOperator._ends_with: "ILIKE", - }.get(op, "=") - - -def __is_negation_operator(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._is_not, - schemas.SearchEventOperator._not_on, - schemas.SearchEventOperator._not_contains] - - -def __reverse_sql_operator(op): - return "=" if op == "!=" else "!=" if op == "=" else "ILIKE" if op == "NOT ILIKE" else "NOT ILIKE" - - -def __get_sql_operator_multiple(op: schemas.SearchEventOperator): - return " IN " if op not in [schemas.SearchEventOperator._is_not, schemas.SearchEventOperator._not_on, - schemas.SearchEventOperator._not_contains] else " NOT IN " - - -def __get_sql_value_multiple(values): - if isinstance(values, tuple): - return values - return tuple(values) if isinstance(values, list) else (values,) - - -def _multiple_conditions(condition, values, value_key="value", is_not=False): - query = [] - for i in range(len(values)): - k = f"{value_key}_{i}" - query.append(condition.replace(value_key, k)) - return "(" + (" AND " if is_not else " OR ").join(query) + ")" - - -def _multiple_values(values, value_key="value"): - query_values = {} - if values is not None and isinstance(values, list): - for i in range(len(values)): - k = f"{value_key}_{i}" - query_values[k] = values[i] - return query_values - - -def _isAny_opreator(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any] - - -def _isUndefined_operator(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._is_undefined] - - # This function executes the query and return result def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False): @@ -261,9 +201,9 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ ORDER BY s.session_id desc) AS filtred_sessions ORDER BY {sort} {data.order}, issue_score DESC) AS full_sessions;""", full_args) - # print("--------------------") - # print(main_query) - # print("--------------------") + print("--------------------") + print(main_query) + print("--------------------") try: cur.execute(main_query) except Exception as err: @@ -438,15 +378,15 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr filter_type = f.type f.value = helper.values_for_operator(value=f.value, op=f.operator) f_k = f"f_value{i}" - full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} - op = __get_sql_operator(f.operator) \ + full_args = {**full_args, **sh.multi_values(f.value, value_key=f_k)} + op = sh.get_sql_operator(f.operator) \ if filter_type not in [schemas.FilterType.events_count] else f.operator - is_any = _isAny_opreator(f.operator) - is_undefined = _isUndefined_operator(f.operator) + is_any = sh.isAny_opreator(f.operator) + is_undefined = sh.isUndefined_operator(f.operator) if not is_any and not is_undefined and len(f.value) == 0: continue is_not = False - if __is_negation_operator(f.operator): + if sh.is_negation_operator(f.operator): is_not = True if filter_type == schemas.FilterType.user_browser: if is_any: @@ -454,9 +394,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_browser IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: if is_any: @@ -464,9 +405,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_os IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: if is_any: @@ -474,9 +415,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_device IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: if is_any: @@ -484,9 +425,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_country IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.utm_source]: if is_any: @@ -497,11 +439,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.utm_source IS NULL') else: extra_constraints.append( - _multiple_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.utm_medium]: if is_any: extra_constraints.append('s.utm_medium IS NOT NULL') @@ -511,11 +453,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.utm_medium IS NULL') else: extra_constraints.append( - _multiple_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.utm_campaign]: if is_any: extra_constraints.append('s.utm_campaign IS NOT NULL') @@ -525,11 +467,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.utm_campaign IS NULL') else: extra_constraints.append( - _multiple_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) elif filter_type == schemas.FilterType.duration: if len(f.value) > 0 and f.value[0] is not None: @@ -546,7 +488,8 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr extra_constraints.append('s.base_referrer IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) elif filter_type == events.event_type.METADATA.ui_type: # get metadata list only if you need it if meta_keys is None: @@ -561,11 +504,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NULL") else: extra_constraints.append( - _multiple_conditions( + sh.multi_conditions( f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions( + sh.multi_conditions( f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: @@ -577,9 +520,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_id IS NULL') else: extra_constraints.append( - _multiple_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.user_anonymous_id, schemas.FilterType.user_anonymous_id_ios]: if is_any: @@ -590,11 +535,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_anonymous_id IS NULL') else: extra_constraints.append( - _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: if is_any: extra_constraints.append('s.rev_id IS NOT NULL') @@ -604,40 +549,58 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.rev_id IS NULL') else: extra_constraints.append( - _multiple_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) elif filter_type == schemas.FilterType.platform: - # op = __get_sql_operator(f.operator) + # op = __ sh.get_sql_operator(f.operator) extra_constraints.append( - _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) elif filter_type == schemas.FilterType.issue: if is_any: extra_constraints.append("array_length(s.issue_types, 1) > 0") ss_constraints.append("array_length(ms.issue_types, 1) > 0") else: extra_constraints.append( - _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + # search sessions with click_rage on a specific selector + if len(f.filters) > 0 and schemas.IssueType.click_rage in f.value: + for j, sf in enumerate(f.filters): + if sf.operator == schemas.IssueFilterOperator._on_selector: + f_k = f"f_value{i}_{j}" + full_args = {**full_args, **sh.multi_values(sf.value, value_key=f_k)} + extra_constraints += ["mc.timestamp>=%(startDate)s", + "mc.timestamp<=%(endDate)s", + "mis.type='click_rage'", + sh.multi_conditions(f"mc.selector=%({f_k})s", + sf.value, is_not=is_not, + value_key=f_k)] + + extra_from += """INNER JOIN events.clicks AS mc USING(session_id) + INNER JOIN events_common.issues USING (session_id,timestamp) + INNER JOIN public.issues AS mis USING (issue_id)\n""" + elif filter_type == schemas.FilterType.events_count: extra_constraints.append( - _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) # --------------------------------------------------------------------------- if len(data.events) > 0: valid_events_count = 0 for event in data.events: - is_any = _isAny_opreator(event.operator) + is_any = sh.isAny_opreator(event.operator) if not isinstance(event.value, list): event.value = [event.value] if __is_valid_event(is_any=is_any, event=event): @@ -649,16 +612,16 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " for i, event in enumerate(data.events): event_type = event.type - is_any = _isAny_opreator(event.operator) + is_any = sh.isAny_opreator(event.operator) if not isinstance(event.value, list): event.value = [event.value] if not __is_valid_event(is_any=is_any, event=event): continue - op = __get_sql_operator(event.operator) + op = sh.get_sql_operator(event.operator) is_not = False - if __is_negation_operator(event.operator): + if sh.is_negation_operator(event.operator): is_not = True - op = __reverse_sql_operator(op) + op = sh.reverse_sql_operator(op) if event_index == 0 or or_events: event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", @@ -678,49 +641,49 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr if event.type != schemas.PerformanceEventType.time_between_events: event.value = helper.values_for_operator(value=event.value, op=event.operator) full_args = {**full_args, - **_multiple_values(event.value, value_key=e_k), - **_multiple_values(event.source, value_key=s_k)} + **sh.multi_values(event.value, value_key=e_k), + **sh.multi_values(event.source, value_key=s_k)} if event_type == events.event_type.CLICK.ui_type: event_from = event_from % f"{events.event_type.CLICK.table} AS main " if not is_any: if event.operator == schemas.ClickEventExtraOperator._on_selector: event_where.append( - _multiple_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k)) + sh.multi_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k)) else: event_where.append( - _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, - value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) elif event_type == events.event_type.INPUT.ui_type: event_from = event_from % f"{events.event_type.INPUT.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, - value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) if event.source is not None and len(event.source) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, - value_key=f"custom{i}")) - full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")} + event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key=f"custom{i}")) + full_args = {**full_args, **sh.multi_values(event.source, value_key=f"custom{i}")} elif event_type == events.event_type.LOCATION.ui_type: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.CUSTOM.ui_type: event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, - value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) elif event_type == events.event_type.REQUEST.ui_type: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, - value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) # elif event_type == events.event_type.GRAPHQL.ui_type: # event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " # if not is_any: @@ -731,17 +694,17 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.ERROR.ui_type: event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" event.source = list(set(event.source)) if not is_any and event.value not in [None, "*", ""]: event_where.append( - _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", - event.value, value_key=e_k)) + sh.multi_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) if len(event.source) > 0 and event.source[0] not in [None, "*", ""]: - event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k)) + event_where.append(sh.multi_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k)) # ----- IOS @@ -749,49 +712,49 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.INPUT_IOS.ui_type: event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) if event.source is not None and len(event.source) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, - value_key="custom{i}")) - full_args = {**full_args, **_multiple_values(event.source, f"custom{i}")} + event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key="custom{i}")) + full_args = {**full_args, **sh.multi_values(event.source, f"custom{i}")} elif event_type == events.event_type.VIEW_IOS.ui_type: event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.CUSTOM_IOS.ui_type: event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.REQUEST_IOS.ui_type: event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.ERROR_IOS.ui_type: event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" if not is_any and event.value not in [None, "*", ""]: event_where.append( - _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", - event.value, value_key=e_k)) + sh.multi_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) elif event_type == schemas.PerformanceEventType.fetch_failed: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) col = performance_event.get_col(event_type) colname = col["column"] event_where.append(f"main.{colname} = FALSE") @@ -805,7 +768,7 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr # colname = col["column"] # tname = "main" # e_k += "_custom" - # full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + # full_args = {**full_args, **_ sh.multiple_values(event.source, value_key=e_k)} # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + # _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", # event.source, value_key=e_k)) @@ -826,14 +789,14 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr f"{tname}.timestamp <= %(endDate)s"] if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)} event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", - event.source, value_key=e_k)) + sh.multi_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) elif event_type == schemas.PerformanceEventType.time_between_events: event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " if not isinstance(event.value[0].value, list): @@ -847,70 +810,71 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr e_k1 = e_k + "_e1" e_k2 = e_k + "_e2" full_args = {**full_args, - **_multiple_values(event.value[0].value, value_key=e_k1), - **_multiple_values(event.value[1].value, value_key=e_k2)} - s_op = __get_sql_operator(event.value[0].operator) + **sh.multi_values(event.value[0].value, value_key=e_k1), + **sh.multi_values(event.value[1].value, value_key=e_k2)} + s_op = sh.get_sql_operator(event.value[0].operator) event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] if event_index > 0 and not or_events: event_where.append("main2.session_id=event_0.session_id") - is_any = _isAny_opreator(event.value[0].operator) + is_any = sh.isAny_opreator(event.value[0].operator) if not is_any: event_where.append( - _multiple_conditions( + sh.multi_conditions( f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", event.value[0].value, value_key=e_k1)) - s_op = __get_sql_operator(event.value[1].operator) - is_any = _isAny_opreator(event.value[1].operator) + s_op = sh.get_sql_operator(event.value[1].operator) + is_any = sh.isAny_opreator(event.value[1].operator) if not is_any: event_where.append( - _multiple_conditions( + sh.multi_conditions( f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", event.value[1].value, value_key=e_k2)) e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)} event_where.append( - _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", - event.source, value_key=e_k)) + sh.multi_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) elif event_type == schemas.EventType.request_details: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " apply = False for j, f in enumerate(event.filters): - is_any = _isAny_opreator(f.operator) + is_any = sh.isAny_opreator(f.operator) if is_any or len(f.value) == 0: continue f.value = helper.values_for_operator(value=f.value, op=f.operator) - op = __get_sql_operator(f.operator) + op = sh.get_sql_operator(f.operator) e_k_f = e_k + f"_fetch{j}" - full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)} if f.type == schemas.FetchFilterType._url: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text", - f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text", + f.value, value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._status_code: event_where.append( - _multiple_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value, - value_key=e_k_f)) + sh.multi_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._method: event_where.append( - _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._duration: event_where.append( - _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, - value_key=e_k_f)) + sh.multi_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._request_body: event_where.append( - _multiple_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, + value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._response_body: event_where.append( - _multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, - value_key=e_k_f)) + sh.multi_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, + value_key=e_k_f)) apply = True else: print(f"undefined FETCH filter: {f.type}") @@ -919,26 +883,26 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr elif event_type == schemas.EventType.graphql: event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " for j, f in enumerate(event.filters): - is_any = _isAny_opreator(f.operator) + is_any = sh.isAny_opreator(f.operator) if is_any or len(f.value) == 0: continue f.value = helper.values_for_operator(value=f.value, op=f.operator) - op = __get_sql_operator(f.operator) + op = sh.get_sql_operator(f.operator) e_k_f = e_k + f"_graphql{j}" - full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)} if f.type == schemas.GraphqlFilterType._name: event_where.append( - _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, - value_key=e_k_f)) + sh.multi_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) elif f.type == schemas.GraphqlFilterType._method: event_where.append( - _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) elif f.type == schemas.GraphqlFilterType._request_body: event_where.append( - _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) elif f.type == schemas.GraphqlFilterType._response_body: event_where.append( - _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) else: print(f"undefined GRAPHQL filter: {f.type}") else: diff --git a/api/chalicelib/core/sessions_notes.py b/api/chalicelib/core/sessions_notes.py index 15eb9369f..dd2de83f3 100644 --- a/api/chalicelib/core/sessions_notes.py +++ b/api/chalicelib/core/sessions_notes.py @@ -2,6 +2,7 @@ from urllib.parse import urljoin from decouple import config +import chalicelib.utils.sql_helper import schemas from chalicelib.core import sessions from chalicelib.core.collaboration_slack import Slack @@ -58,8 +59,8 @@ def get_all_notes_by_project_id(tenant_id, project_id, user_id, data: schemas.Se if data.tags and len(data.tags) > 0: k = "tag_value" conditions.append( - sessions._multiple_conditions(f"%({k})s = sessions_notes.tag", data.tags, value_key=k)) - extra_params = sessions._multiple_values(data.tags, value_key=k) + chalicelib.utils.sql_helper.multi_conditions(f"%({k})s = sessions_notes.tag", data.tags, value_key=k)) + extra_params = chalicelib.utils.sql_helper.multi_values(data.tags, value_key=k) if data.shared_only: conditions.append("sessions_notes.is_public") elif data.mine_only: diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 86528eea8..43d9d2fa9 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -1,6 +1,7 @@ __author__ = "AZNAUROV David" __maintainer__ = "KRAIEM Taha Yassine" +import chalicelib.utils.sql_helper import schemas from chalicelib.core import events, metadata, sessions @@ -49,33 +50,33 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: continue f["value"] = helper.values_for_operator(value=f["value"], op=f["operator"]) # filter_args = _multiple_values(f["value"]) - op = sessions.__get_sql_operator(f["operator"]) + op = chalicelib.utils.sql_helper.get_sql_operator(f["operator"]) filter_type = f["type"] # values[f_k] = sessions.__get_sql_value_multiple(f["value"]) f_k = f"f_value{i}" values = {**values, - **sessions._multiple_values(helper.values_for_operator(value=f["value"], op=f["operator"]), - value_key=f_k)} + **chalicelib.utils.sql_helper.multi_values(helper.values_for_operator(value=f["value"], op=f["operator"]), + value_key=f_k)} if filter_type == schemas.FilterType.user_browser: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sessions._multiple_conditions(f's.user_browser {op} %({f_k})s', f["value"], value_key=f_k)) + chalicelib.utils.sql_helper.multi_conditions(f's.user_browser {op} %({f_k})s', f["value"], value_key=f_k)) elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sessions._multiple_conditions(f's.user_os {op} %({f_k})s', f["value"], value_key=f_k)) + chalicelib.utils.sql_helper.multi_conditions(f's.user_os {op} %({f_k})s', f["value"], value_key=f_k)) elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sessions._multiple_conditions(f's.user_device {op} %({f_k})s', f["value"], value_key=f_k)) + chalicelib.utils.sql_helper.multi_conditions(f's.user_device {op} %({f_k})s', f["value"], value_key=f_k)) elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sessions._multiple_conditions(f's.user_country {op} %({f_k})s', f["value"], value_key=f_k)) + chalicelib.utils.sql_helper.multi_conditions(f's.user_country {op} %({f_k})s', f["value"], value_key=f_k)) elif filter_type == schemas.FilterType.duration: if len(f["value"]) > 0 and f["value"][0] is not None: first_stage_extra_constraints.append(f's.duration >= %(minDuration)s') @@ -88,7 +89,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: filter_extra_from = [f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)"] # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sessions._multiple_conditions(f"p.base_referrer {op} %({f_k})s", f["value"], value_key=f_k)) + chalicelib.utils.sql_helper.multi_conditions(f"p.base_referrer {op} %({f_k})s", f["value"], value_key=f_k)) elif filter_type == events.event_type.METADATA.ui_type: if meta_keys is None: meta_keys = metadata.get(project_id=project_id) @@ -96,25 +97,25 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: # op = sessions.__get_sql_operator(f["operator"]) if f.get("key") in meta_keys.keys(): first_stage_extra_constraints.append( - sessions._multiple_conditions( + chalicelib.utils.sql_helper.multi_conditions( f's.{metadata.index_to_colname(meta_keys[f["key"]])} {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( - sessions._multiple_conditions(f's.user_id {op} %({f_k})s', f["value"], value_key=f_k)) + chalicelib.utils.sql_helper.multi_conditions(f's.user_id {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [schemas.FilterType.user_anonymous_id, schemas.FilterType.user_anonymous_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( - sessions._multiple_conditions(f's.user_anonymous_id {op} %({f_k})s', f["value"], value_key=f_k)) + chalicelib.utils.sql_helper.multi_conditions(f's.user_anonymous_id {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( - sessions._multiple_conditions(f's.rev_id {op} %({f_k})s', f["value"], value_key=f_k)) + chalicelib.utils.sql_helper.multi_conditions(f's.rev_id {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) i = -1 for s in stages: @@ -124,7 +125,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: if not isinstance(s["value"], list): s["value"] = [s["value"]] - is_any = sessions._isAny_opreator(s["operator"]) + is_any = chalicelib.utils.sql_helper.isAny_opreator(s["operator"]) if not is_any and isinstance(s["value"], list) and len(s["value"]) == 0: continue i += 1 @@ -132,7 +133,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: extra_from = filter_extra_from + ["INNER JOIN public.sessions AS s USING (session_id)"] else: extra_from = [] - op = sessions.__get_sql_operator(s["operator"]) + op = chalicelib.utils.sql_helper.get_sql_operator(s["operator"]) event_type = s["type"].upper() if event_type == events.event_type.CLICK.ui_type: next_table = events.event_type.CLICK.table @@ -163,10 +164,10 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: print("=================UNDEFINED") continue - values = {**values, **sessions._multiple_values(helper.values_for_operator(value=s["value"], op=s["operator"]), - value_key=f"value{i + 1}")} - if sessions.__is_negation_operator(op) and i > 0: - op = sessions.__reverse_sql_operator(op) + values = {**values, **chalicelib.utils.sql_helper.multi_values(helper.values_for_operator(value=s["value"], op=s["operator"]), + value_key=f"value{i + 1}")} + if chalicelib.utils.sql_helper.is_negation_operator(op) and i > 0: + op = chalicelib.utils.sql_helper.reverse_sql_operator(op) main_condition = "left_not.session_id ISNULL" extra_from.append(f"""LEFT JOIN LATERAL (SELECT session_id FROM {next_table} AS s_main @@ -177,8 +178,8 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: if is_any: main_condition = "TRUE" else: - main_condition = sessions._multiple_conditions(f"main.{next_col_name} {op} %(value{i + 1})s", - values=s["value"], value_key=f"value{i + 1}") + main_condition = chalicelib.utils.sql_helper.multi_conditions(f"main.{next_col_name} {op} %(value{i + 1})s", + values=s["value"], value_key=f"value{i + 1}") n_stages_query.append(f""" (SELECT main.session_id, {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp diff --git a/api/schemas.py b/api/schemas.py index 6cf162e8a..045ef3711 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -445,6 +445,10 @@ class ClickEventExtraOperator(str, Enum): _on_text = "onText" +class IssueFilterOperator(str, Enum): + _on_selector = ClickEventExtraOperator._on_selector.value + + class PlatformType(str, Enum): mobile = "mobile" desktop = "desktop" @@ -520,20 +524,30 @@ class GraphqlFilterType(str, Enum): _response_body = "GRAPHQL_RESPONSE_BODY" +class IssueFilterType(str, Enum): + _selector = "CLICK_SELECTOR" + + class RequestGraphqlFilterSchema(BaseModel): type: Union[FetchFilterType, GraphqlFilterType] = Field(...) value: List[Union[int, str]] = Field(...) operator: Union[SearchEventOperator, MathOperator] = Field(...) +class IssueFilterSchema(BaseModel): + type: IssueFilterType = Field(...) + value: List[str] = Field(...) + operator: IssueFilterOperator = Field(...) + + class _SessionSearchEventRaw(__MixedSearchFilter): is_event: bool = Field(default=True, const=True) value: List[str] = Field(...) type: Union[EventType, PerformanceEventType] = Field(...) operator: Union[SearchEventOperator, ClickEventExtraOperator] = Field(...) - source: Optional[List[Union[ErrorSource, int, str]]] = Field(None) - sourceOperator: Optional[MathOperator] = Field(None) - filters: Optional[List[RequestGraphqlFilterSchema]] = Field(None) + source: Optional[List[Union[ErrorSource, int, str]]] = Field(default=None) + sourceOperator: Optional[MathOperator] = Field(default=None) + filters: Optional[List[Union[RequestGraphqlFilterSchema, IssueFilterSchema]]] = Field(default=None) @root_validator def event_validator(cls, values): @@ -581,11 +595,14 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): class SessionSearchFilterSchema(__MixedSearchFilter): is_event: bool = Field(False, const=False) - value: Union[Optional[Union[IssueType, PlatformType, int, str]], - Optional[List[Union[IssueType, PlatformType, int, str]]]] = Field(...) + # TODO: remove this if there nothing broken from the UI + # value: Union[Optional[Union[IssueType, PlatformType, int, str]], + # Optional[List[Union[IssueType, PlatformType, int, str]]]] = Field(...) + value: List[Union[IssueType, PlatformType, int, str]] = Field(default=[]) type: FilterType = Field(...) operator: Union[SearchEventOperator, MathOperator] = Field(...) source: Optional[Union[ErrorSource, str]] = Field(default=None) + filters: List[IssueFilterSchema] = Field(default=[]) @root_validator def filter_validator(cls, values): @@ -1215,12 +1232,8 @@ class FlatClickMapSessionsSearch(SessionsSearchPayloadSchema): return values -class IssueFilterType(str, Enum): - _on_selector = ClickEventExtraOperator._on_selector.value - - class IssueAdvancedFilter(BaseModel): - type: IssueFilterType = Field(default=IssueFilterType._on_selector) + type: IssueFilterType = Field(default=IssueFilterType._selector) value: List[str] = Field(default=[]) operator: SearchEventOperator = Field(default=SearchEventOperator._is) diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 194373eeb..95afa95ea 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -236,6 +236,7 @@ Pipfile /chalicelib/utils/pg_client.py /chalicelib/utils/s3.py /chalicelib/utils/smtp.py +/chalicelib/utils/sql_helper.py /chalicelib/utils/strings.py /chalicelib/utils/TimeUTC.py /routers/app/__init__.py diff --git a/ee/api/chalicelib/core/autocomplete_exp.py b/ee/api/chalicelib/core/autocomplete_exp.py index db2ecb95b..2abe97e8e 100644 --- a/ee/api/chalicelib/core/autocomplete_exp.py +++ b/ee/api/chalicelib/core/autocomplete_exp.py @@ -3,7 +3,7 @@ from chalicelib.utils import ch_client from chalicelib.utils import helper from chalicelib.utils.event_filter_definition import Event -TABLE = "final.autocomplete" +TABLE = "experimental.autocomplete" def __get_autocomplete_table(value, project_id): diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py index 0808f4105..b7ff6e937 100644 --- a/ee/api/chalicelib/core/sessions.py +++ b/ee/api/chalicelib/core/sessions.py @@ -6,6 +6,7 @@ from chalicelib.core import events, metadata, events_ios, \ sessions_mobs, issues, projects, errors, resources, assist, performance_event, sessions_viewed, sessions_favorite, \ sessions_devtool, sessions_notes from chalicelib.utils import pg_client, helper, metrics_helper +from chalicelib.utils import sql_helper as sh SESSION_PROJECTION_COLS = """s.project_id, s.session_id::text AS session_id, @@ -117,67 +118,6 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu return None -def __get_sql_operator(op: Union[schemas.SearchEventOperator, schemas.ClickEventExtraOperator]): - return { - schemas.SearchEventOperator._is: "=", - schemas.SearchEventOperator._is_any: "IN", - schemas.SearchEventOperator._on: "=", - schemas.SearchEventOperator._on_any: "IN", - schemas.SearchEventOperator._is_not: "!=", - schemas.SearchEventOperator._not_on: "!=", - schemas.SearchEventOperator._contains: "ILIKE", - schemas.SearchEventOperator._not_contains: "NOT ILIKE", - schemas.SearchEventOperator._starts_with: "ILIKE", - schemas.SearchEventOperator._ends_with: "ILIKE", - }.get(op, "=") - - -def __is_negation_operator(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._is_not, - schemas.SearchEventOperator._not_on, - schemas.SearchEventOperator._not_contains] - - -def __reverse_sql_operator(op): - return "=" if op == "!=" else "!=" if op == "=" else "ILIKE" if op == "NOT ILIKE" else "NOT ILIKE" - - -def __get_sql_operator_multiple(op: schemas.SearchEventOperator): - return " IN " if op not in [schemas.SearchEventOperator._is_not, schemas.SearchEventOperator._not_on, - schemas.SearchEventOperator._not_contains] else " NOT IN " - - -def __get_sql_value_multiple(values): - if isinstance(values, tuple): - return values - return tuple(values) if isinstance(values, list) else (values,) - - -def _multiple_conditions(condition, values, value_key="value", is_not=False): - query = [] - for i in range(len(values)): - k = f"{value_key}_{i}" - query.append(condition.replace(value_key, k)) - return "(" + (" AND " if is_not else " OR ").join(query) + ")" - - -def _multiple_values(values, value_key="value"): - query_values = {} - if values is not None and isinstance(values, list): - for i in range(len(values)): - k = f"{value_key}_{i}" - query_values[k] = values[i] - return query_values - - -def _isAny_opreator(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any] - - -def _isUndefined_operator(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._is_undefined] - - # This function executes the query and return result def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False): @@ -441,15 +381,15 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr filter_type = f.type f.value = helper.values_for_operator(value=f.value, op=f.operator) f_k = f"f_value{i}" - full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} - op = __get_sql_operator(f.operator) \ + full_args = {**full_args, **sh.multi_values(f.value, value_key=f_k)} + op = sh.get_sql_operator(f.operator) \ if filter_type not in [schemas.FilterType.events_count] else f.operator - is_any = _isAny_opreator(f.operator) - is_undefined = _isUndefined_operator(f.operator) + is_any = sh.isAny_opreator(f.operator) + is_undefined = sh.isUndefined_operator(f.operator) if not is_any and not is_undefined and len(f.value) == 0: continue is_not = False - if __is_negation_operator(f.operator): + if sh.is_negation_operator(f.operator): is_not = True if filter_type == schemas.FilterType.user_browser: if is_any: @@ -457,9 +397,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_browser IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: if is_any: @@ -467,9 +408,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_os IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: if is_any: @@ -477,9 +418,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_device IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: if is_any: @@ -487,9 +428,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_country IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.utm_source]: if is_any: @@ -500,11 +442,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.utm_source IS NULL') else: extra_constraints.append( - _multiple_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.utm_medium]: if is_any: extra_constraints.append('s.utm_medium IS NOT NULL') @@ -514,11 +456,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.utm_medium IS NULL') else: extra_constraints.append( - _multiple_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.utm_campaign]: if is_any: extra_constraints.append('s.utm_campaign IS NOT NULL') @@ -528,11 +470,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.utm_campaign IS NULL') else: extra_constraints.append( - _multiple_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) elif filter_type == schemas.FilterType.duration: if len(f.value) > 0 and f.value[0] is not None: @@ -549,7 +491,8 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr extra_constraints.append('s.base_referrer IS NOT NULL') else: extra_constraints.append( - _multiple_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) elif filter_type == events.event_type.METADATA.ui_type: # get metadata list only if you need it if meta_keys is None: @@ -564,11 +507,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NULL") else: extra_constraints.append( - _multiple_conditions( + sh.multi_conditions( f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions( + sh.multi_conditions( f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: @@ -580,9 +523,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_id IS NULL') else: extra_constraints.append( - _multiple_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.user_anonymous_id, schemas.FilterType.user_anonymous_id_ios]: if is_any: @@ -593,11 +538,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.user_anonymous_id IS NULL') else: extra_constraints.append( - _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: if is_any: extra_constraints.append('s.rev_id IS NOT NULL') @@ -607,40 +552,41 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr ss_constraints.append('ms.rev_id IS NULL') else: extra_constraints.append( - _multiple_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + sh.multi_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) elif filter_type == schemas.FilterType.platform: - # op = __get_sql_operator(f.operator) + # op = __ sh.get_sql_operator(f.operator) extra_constraints.append( - _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) elif filter_type == schemas.FilterType.issue: if is_any: extra_constraints.append("array_length(s.issue_types, 1) > 0") ss_constraints.append("array_length(ms.issue_types, 1) > 0") else: extra_constraints.append( - _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) elif filter_type == schemas.FilterType.events_count: extra_constraints.append( - _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) + sh.multi_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) # --------------------------------------------------------------------------- if len(data.events) > 0: valid_events_count = 0 for event in data.events: - is_any = _isAny_opreator(event.operator) + is_any = sh.isAny_opreator(event.operator) if not isinstance(event.value, list): event.value = [event.value] if __is_valid_event(is_any=is_any, event=event): @@ -652,16 +598,16 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " for i, event in enumerate(data.events): event_type = event.type - is_any = _isAny_opreator(event.operator) + is_any = sh.isAny_opreator(event.operator) if not isinstance(event.value, list): event.value = [event.value] if not __is_valid_event(is_any=is_any, event=event): continue - op = __get_sql_operator(event.operator) + op = sh.get_sql_operator(event.operator) is_not = False - if __is_negation_operator(event.operator): + if sh.is_negation_operator(event.operator): is_not = True - op = __reverse_sql_operator(op) + op = sh.reverse_sql_operator(op) if event_index == 0 or or_events: event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", @@ -681,49 +627,49 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr if event.type != schemas.PerformanceEventType.time_between_events: event.value = helper.values_for_operator(value=event.value, op=event.operator) full_args = {**full_args, - **_multiple_values(event.value, value_key=e_k), - **_multiple_values(event.source, value_key=s_k)} + **sh.multi_values(event.value, value_key=e_k), + **sh.multi_values(event.source, value_key=s_k)} if event_type == events.event_type.CLICK.ui_type: event_from = event_from % f"{events.event_type.CLICK.table} AS main " if not is_any: if event.operator == schemas.ClickEventExtraOperator._on_selector: event_where.append( - _multiple_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k)) + sh.multi_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k)) else: event_where.append( - _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, - value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) elif event_type == events.event_type.INPUT.ui_type: event_from = event_from % f"{events.event_type.INPUT.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, - value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) if event.source is not None and len(event.source) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, - value_key=f"custom{i}")) - full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")} + event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key=f"custom{i}")) + full_args = {**full_args, **sh.multi_values(event.source, value_key=f"custom{i}")} elif event_type == events.event_type.LOCATION.ui_type: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.CUSTOM.ui_type: event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, - value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) elif event_type == events.event_type.REQUEST.ui_type: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, - value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) # elif event_type == events.event_type.GRAPHQL.ui_type: # event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " # if not is_any: @@ -734,17 +680,17 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.ERROR.ui_type: event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" event.source = list(set(event.source)) if not is_any and event.value not in [None, "*", ""]: event_where.append( - _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", - event.value, value_key=e_k)) + sh.multi_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) if len(event.source) > 0 and event.source[0] not in [None, "*", ""]: - event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k)) + event_where.append(sh.multi_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k)) # ----- IOS @@ -752,49 +698,49 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.INPUT_IOS.ui_type: event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) if event.source is not None and len(event.source) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, - value_key="custom{i}")) - full_args = {**full_args, **_multiple_values(event.source, f"custom{i}")} + event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key="custom{i}")) + full_args = {**full_args, **sh.multi_values(event.source, f"custom{i}")} elif event_type == events.event_type.VIEW_IOS.ui_type: event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.CUSTOM_IOS.ui_type: event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.REQUEST_IOS.ui_type: event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.ERROR_IOS.ui_type: event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" if not is_any and event.value not in [None, "*", ""]: event_where.append( - _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", - event.value, value_key=e_k)) + sh.multi_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) elif event_type == schemas.PerformanceEventType.fetch_failed: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) col = performance_event.get_col(event_type) colname = col["column"] event_where.append(f"main.{colname} = FALSE") @@ -808,7 +754,7 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr # colname = col["column"] # tname = "main" # e_k += "_custom" - # full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + # full_args = {**full_args, **_ sh.multiple_values(event.source, value_key=e_k)} # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + # _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", # event.source, value_key=e_k)) @@ -829,14 +775,14 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr f"{tname}.timestamp <= %(endDate)s"] if not is_any: event_where.append( - _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) + sh.multi_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)} event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", - event.source, value_key=e_k)) + sh.multi_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) elif event_type == schemas.PerformanceEventType.time_between_events: event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " if not isinstance(event.value[0].value, list): @@ -850,70 +796,71 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr e_k1 = e_k + "_e1" e_k2 = e_k + "_e2" full_args = {**full_args, - **_multiple_values(event.value[0].value, value_key=e_k1), - **_multiple_values(event.value[1].value, value_key=e_k2)} - s_op = __get_sql_operator(event.value[0].operator) + **sh.multi_values(event.value[0].value, value_key=e_k1), + **sh.multi_values(event.value[1].value, value_key=e_k2)} + s_op = sh.get_sql_operator(event.value[0].operator) event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] if event_index > 0 and not or_events: event_where.append("main2.session_id=event_0.session_id") - is_any = _isAny_opreator(event.value[0].operator) + is_any = sh.isAny_opreator(event.value[0].operator) if not is_any: event_where.append( - _multiple_conditions( + sh.multi_conditions( f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", event.value[0].value, value_key=e_k1)) - s_op = __get_sql_operator(event.value[1].operator) - is_any = _isAny_opreator(event.value[1].operator) + s_op = sh.get_sql_operator(event.value[1].operator) + is_any = sh.isAny_opreator(event.value[1].operator) if not is_any: event_where.append( - _multiple_conditions( + sh.multi_conditions( f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", event.value[1].value, value_key=e_k2)) e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)} event_where.append( - _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", - event.source, value_key=e_k)) + sh.multi_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) elif event_type == schemas.EventType.request_details: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " apply = False for j, f in enumerate(event.filters): - is_any = _isAny_opreator(f.operator) + is_any = sh.isAny_opreator(f.operator) if is_any or len(f.value) == 0: continue f.value = helper.values_for_operator(value=f.value, op=f.operator) - op = __get_sql_operator(f.operator) + op = sh.get_sql_operator(f.operator) e_k_f = e_k + f"_fetch{j}" - full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)} if f.type == schemas.FetchFilterType._url: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text", - f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text", + f.value, value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._status_code: event_where.append( - _multiple_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value, - value_key=e_k_f)) + sh.multi_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._method: event_where.append( - _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._duration: event_where.append( - _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, - value_key=e_k_f)) + sh.multi_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._request_body: event_where.append( - _multiple_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, + value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._response_body: event_where.append( - _multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, - value_key=e_k_f)) + sh.multi_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, + value_key=e_k_f)) apply = True else: print(f"undefined FETCH filter: {f.type}") @@ -922,26 +869,26 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr elif event_type == schemas.EventType.graphql: event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " for j, f in enumerate(event.filters): - is_any = _isAny_opreator(f.operator) + is_any = sh.isAny_opreator(f.operator) if is_any or len(f.value) == 0: continue f.value = helper.values_for_operator(value=f.value, op=f.operator) - op = __get_sql_operator(f.operator) + op = sh.get_sql_operator(f.operator) e_k_f = e_k + f"_graphql{j}" - full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)} if f.type == schemas.GraphqlFilterType._name: event_where.append( - _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, - value_key=e_k_f)) + sh.multi_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) elif f.type == schemas.GraphqlFilterType._method: event_where.append( - _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) elif f.type == schemas.GraphqlFilterType._request_body: event_where.append( - _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) elif f.type == schemas.GraphqlFilterType._response_body: event_where.append( - _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + sh.multi_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) else: print(f"undefined GRAPHQL filter: {f.type}") else: diff --git a/ee/api/clean.sh b/ee/api/clean.sh index 754193f95..49b803d0f 100755 --- a/ee/api/clean.sh +++ b/ee/api/clean.sh @@ -58,6 +58,7 @@ rm -rf ./chalicelib/utils/metrics_helper.py rm -rf ./chalicelib/utils/pg_client.py rm -rf ./chalicelib/utils/s3.py rm -rf ./chalicelib/utils/smtp.py +rm -rf ./chalicelib/utils/sql_helper.py rm -rf ./chalicelib/utils/strings.py rm -rf ./chalicelib/utils/TimeUTC.py rm -rf ./routers/app/__init__.py diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql index a53ca073d..494625936 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql @@ -91,6 +91,22 @@ DROP TYPE IF EXISTS metric_view_type; ALTER TABLE IF EXISTS events.clicks ADD COLUMN IF NOT EXISTS path text; +DROP INDEX IF EXISTS events.inputs_label_value_idx; +DROP INDEX IF EXISTS events.inputs_label_idx; +DROP INDEX IF EXISTS events.pages_base_path_idx2; +DROP INDEX IF EXISTS events.pages_base_referrer_gin_idx2; +DROP INDEX IF EXISTS events.resources_url_gin_idx; +DROP INDEX IF EXISTS events.resources_url_idx; +DROP INDEX IF EXISTS events.resources_url_hostpath_idx; +DROP INDEX IF EXISTS events.resources_session_id_timestamp_idx; +DROP INDEX IF EXISTS events.resources_duration_durationgt0_idx; +DROP INDEX IF EXISTS events.state_actions_name_idx; +DROP INDEX IF EXISTS events_common.requests_query_nn_idx; +DROP INDEX IF EXISTS events_common.requests_host_nn_idx; +DROP INDEX IF EXISTS public.sessions_user_country_gin_idx; +DROP INDEX IF EXISTS public.sessions_user_browser_gin_idx; +DROP INDEX IF EXISTS public.sessions_user_os_gin_idx; + COMMIT; CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); diff --git a/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql b/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql index b56a8836c..f7608947b 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql @@ -78,6 +78,23 @@ DROP TYPE IF EXISTS metric_view_type; ALTER TABLE IF EXISTS events.clicks ADD COLUMN IF NOT EXISTS path text; +DROP INDEX IF EXISTS events.inputs_label_value_idx; +DROP INDEX IF EXISTS events.inputs_label_idx; +DROP INDEX IF EXISTS events.pages_base_path_idx2; +DROP INDEX IF EXISTS events.pages_base_referrer_gin_idx2; +DROP INDEX IF EXISTS events.resources_url_gin_idx; +DROP INDEX IF EXISTS events.resources_url_idx; +DROP INDEX IF EXISTS events.resources_url_hostpath_idx; +DROP INDEX IF EXISTS events.resources_session_id_timestamp_idx; +DROP INDEX IF EXISTS events.resources_duration_durationgt0_idx; +DROP INDEX IF EXISTS events.state_actions_name_idx; +DROP INDEX IF EXISTS events_common.requests_query_nn_idx; +DROP INDEX IF EXISTS events_common.requests_host_nn_idx; +DROP INDEX IF EXISTS public.sessions_user_country_gin_idx; +DROP INDEX IF EXISTS public.sessions_user_browser_gin_idx; +DROP INDEX IF EXISTS public.sessions_user_os_gin_idx; + + COMMIT; CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector);