diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index 69213a079..deb16cb8b 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -1,6 +1,7 @@ -from chalicelib.utils import pg_client, helper -from chalicelib.core import sessions_metas, metadata +import schemas from chalicelib.core import issues +from chalicelib.core import sessions_metas, metadata +from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC from chalicelib.utils.event_filter_definition import SupportedFilter, Event @@ -235,23 +236,23 @@ def __generic_autocomplete(event: Event): class event_type: - CLICK = Event(ui_type="CLICK", table="events.clicks", column="label") - INPUT = Event(ui_type="INPUT", table="events.inputs", column="label") - LOCATION = Event(ui_type="LOCATION", table="events.pages", column="base_path") - CUSTOM = Event(ui_type="CUSTOM", table="events_common.customs", column="name") - REQUEST = Event(ui_type="REQUEST", table="events_common.requests", column="url") - GRAPHQL = Event(ui_type="GRAPHQL", table="events.graphql", column="name") - STATEACTION = Event(ui_type="STATEACTION", table="events.state_actions", column="name") - ERROR = Event(ui_type="ERROR", table="events.errors", + CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label") + INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label") + LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="base_path") + CUSTOM = Event(ui_type=schemas.EventType.custom, table="events_common.customs", column="name") + REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="url") + GRAPHQL = Event(ui_type=schemas.EventType.graphql, table="events.graphql", column="name") + STATEACTION = Event(ui_type=schemas.EventType.state_action, table="events.state_actions", column="name") + ERROR = Event(ui_type=schemas.EventType.error, table="events.errors", column=None) # column=None because errors are searched by name or message - METADATA = Event(ui_type="METADATA", table="public.sessions", column=None) + METADATA = Event(ui_type=schemas.EventType.metadata, table="public.sessions", column=None) # IOS - CLICK_IOS = Event(ui_type="CLICK_IOS", table="events_ios.clicks", column="label") - INPUT_IOS = Event(ui_type="INPUT_IOS", table="events_ios.inputs", column="label") - VIEW_IOS = Event(ui_type="VIEW_IOS", table="events_ios.views", column="name") - CUSTOM_IOS = Event(ui_type="CUSTOM_IOS", table="events_common.customs", column="name") - REQUEST_IOS = Event(ui_type="REQUEST_IOS", table="events_common.requests", column="url") - ERROR_IOS = Event(ui_type="ERROR_IOS", table="events_ios.crashes", + CLICK_IOS = Event(ui_type=schemas.EventType.click_ios, table="events_ios.clicks", column="label") + INPUT_IOS = Event(ui_type=schemas.EventType.input_ios, table="events_ios.inputs", column="label") + VIEW_IOS = Event(ui_type=schemas.EventType.view_ios, table="events_ios.views", column="name") + CUSTOM_IOS = Event(ui_type=schemas.EventType.custom_ios, table="events_common.customs", column="name") + REQUEST_IOS = Event(ui_type=schemas.EventType.request_ios, table="events_common.requests", column="url") + ERROR_IOS = Event(ui_type=schemas.EventType.error_ios, table="events_ios.crashes", column=None) # column=None because errors are searched by name or message diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index a6d5a50e0..ee732c848 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -109,7 +109,6 @@ def __is_multivalue(op: schemas.SearchEventOperator): def __get_sql_operator(op: schemas.SearchEventOperator): - op = op.lower() return { schemas.SearchEventOperator._is: "=", schemas.SearchEventOperator._is_any: "IN", @@ -145,6 +144,22 @@ def __get_sql_value_multiple(values): return tuple(values) if isinstance(values, list) else (values,) +def __multiple_conditions(condition, values, value_key="value"): + query = [] + for i in range(len(values)): + k = f"{value_key}_{i}" + query.append(condition.replace("value", k)) + return "(" + " OR ".join(query) + ")" + + +def __multiple_values(values, value_key="value"): + query_values = {} + for i in range(len(values)): + k = f"{value_key}_{i}" + query_values[k] = values[i] + return query_values + + @dev.timed def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, favorite_only=False, errors_only=False, error_status="ALL", @@ -257,15 +272,17 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints = [s.decode('UTF-8') for s in ss_constraints] events_query_from = [] event_index = 0 - + events_joiner = " FULL JOIN " if data.events_order == schemas.SearchEventOrder._or else " INNER JOIN LATERAL " for event in data.events: event_type = event.type.upper() + if not isinstance(event.value, list): + event.value = [event.value] op = __get_sql_operator(event.operator) is_not = False if __is_negation_operator(event.operator): is_not = True op = __reverse_sql_operator(op) - if event_index == 0: + if event_index == 0 or data.events_order == schemas.SearchEventOrder._or: event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", @@ -273,13 +290,12 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f else: event_from = "%s" event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", - f"event_{event_index - 1}.timestamp <= main.timestamp", "main.session_id=event_0.session_id"] - if __is_multivalue(event.operator): - event_args = {"value": __get_sql_value_multiple(event.value)} - else: - event.value = helper.string_to_op(value=event.value, op=event.operator) - event_args = {"value": helper.string_to_sql_like_with_op(event.value, op)} + if data.events_order == schemas.SearchEventOrder._then: + event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") + + event.value = helper.values_for_operator(value=event.value, op=event.operator) + event_args = __multiple_values(event.value) if event_type not in list(events.SUPPORTED_TYPES.keys()) \ or event.value in [None, "", "*"] \ and (event_type != events.event_type.ERROR.ui_type \ @@ -287,32 +303,50 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f continue if event_type == events.event_type.CLICK.ui_type: event_from = event_from % f"{events.event_type.CLICK.table} AS main " - event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s") elif event_type == events.event_type.INPUT.ui_type: event_from = event_from % f"{events.event_type.INPUT.table} AS main " - event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s") if len(event.custom) > 0: - event_where.append("main.value ILIKE %(custom)s") - event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") + event_where.append(__multiple_conditions(f"main.value ILIKE %(custom)s", + event.custom, value_key="custom")) + event_args = {**event_args, **__multiple_values(event.custom, value_key="custom")} + # event_where.append("main.value ILIKE %(custom)s") + # event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") elif event_type == events.event_type.LOCATION.ui_type: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " - event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s") elif event_type == events.event_type.CUSTOM.ui_type: event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " - event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s") elif event_type == events.event_type.REQUEST.ui_type: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s") elif event_type == events.event_type.GRAPHQL.ui_type: event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " - event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s") elif event_type == events.event_type.STATEACTION.ui_type: event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " - event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s") elif event_type == events.event_type.ERROR.ui_type: - if event.source in [None, "*", ""]: - event.source = "js_exception" + # if event.source in [None, "*", ""]: + # event.source = "js_exception" event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" if event.value not in [None, "*", ""]: event_where.append(f"(main1.message {op} %(value)s OR main1.name {op} %(value)s)") @@ -326,40 +360,55 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # ----- IOS elif event_type == events.event_type.CLICK_IOS.ui_type: event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " - event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s", event.value)) + # event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s") elif event_type == events.event_type.INPUT_IOS.ui_type: event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " - event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s") - + event_where.append( + __multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s", event.value)) + # event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s") if len(event.custom) > 0: - event_where.append("main.value ILIKE %(custom)s") - event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") + event_where.append(__multiple_conditions("main.value ILIKE %(custom)s", event.custom)) + event_args = {**event_args, **__multiple_values(event.custom, "custom")} + # event_where.append("main.value ILIKE %(custom)s") + # event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") elif event_type == events.event_type.VIEW_IOS.ui_type: event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " - event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s", event.value)) + # event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s") elif event_type == events.event_type.CUSTOM_IOS.ui_type: event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " - event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s") elif event_type == events.event_type.REQUEST_IOS.ui_type: event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " - event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s") elif event_type == events.event_type.ERROR_IOS.ui_type: event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" if event.value not in [None, "*", ""]: - event_where.append(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)") + event_where.append( + __multiple_conditions(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)", + event.value)) + # event_where.append(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)") else: continue - if event_index == 0: + if event_index == 0 or data.events_order == schemas.SearchEventOrder._or: event_where += ss_constraints if is_not: if event_index == 0: events_query_from.append(cur.mogrify(f"""\ (SELECT session_id, - 0 AS timestamp, - {event_index} AS funnel_step + 0 AS timestamp FROM sessions WHERE EXISTS(SELECT session_id FROM {event_from} @@ -375,14 +424,13 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f events_query_from.append(cur.mogrify(f"""\ (SELECT event_0.session_id, - event_{event_index - 1}.timestamp AS timestamp, - {event_index} AS funnel_step + event_{event_index - 1}.timestamp AS timestamp WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ """, {**generic_args, **event_args}).decode('UTF-8')) else: events_query_from.append(cur.mogrify(f"""\ - (SELECT main.session_id, MIN(timestamp) AS timestamp,{event_index} AS funnel_step + (SELECT main.session_id, MIN(timestamp) AS timestamp FROM {event_from} WHERE {" AND ".join(event_where)} GROUP BY 1 @@ -394,7 +442,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_0.session_id, MIN(event_0.timestamp) AS first_event_ts, MAX(event_{event_index - 1}.timestamp) AS last_event_ts - FROM {(" INNER JOIN LATERAL ").join(events_query_from)} + FROM {events_joiner.join(events_query_from)} GROUP BY 1 {fav_only_join}""" else: @@ -488,8 +536,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", generic_args) - # print("--------------------") - # print(main_query) + print("--------------------") + print(main_query) cur.execute(main_query) diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index b8b571d03..a9cd6afd3 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -1,6 +1,7 @@ import random import re import string +from typing import Union import math import requests @@ -168,39 +169,56 @@ def string_to_sql_like(value): def string_to_sql_like_with_op(value, op): - if isinstance(value, list) and len(value) > 0: - _value = value[0] + if isinstance(value, list): + r = [] + for v in value: + r.append(string_to_sql_like_with_op(v, op)) + return r else: _value = value - if _value is None: - return _value - if op.upper() != 'ILIKE': + if _value is None: + return _value + if op.upper() != 'ILIKE': + return _value.replace("%", "%%") + _value = _value.replace("*", "%") + if _value.startswith("^"): + _value = _value[1:] + elif not _value.startswith("%"): + _value = '%' + _value + + if _value.endswith("$"): + _value = _value[:-1] + elif not _value.endswith("%"): + _value = _value + '%' return _value.replace("%", "%%") - _value = _value.replace("*", "%") - if _value.startswith("^"): - _value = _value[1:] - elif not _value.startswith("%"): - _value = '%' + _value - - if _value.endswith("$"): - _value = _value[:-1] - elif not _value.endswith("%"): - _value = _value + '%' - return _value.replace("%", "%%") -def string_to_op(value: str, op: schemas.SearchEventOperator): - if isinstance(value, list) and len(value) > 0: - _value = value[0] +likable_operators = [schemas.SearchEventOperator._starts_with, schemas.SearchEventOperator._ends_with, + schemas.SearchEventOperator._contains, schemas.SearchEventOperator._notcontains] + + +def is_likable(op: schemas.SearchEventOperator): + return op in likable_operators + + +def values_for_operator(value: Union[str, list], op: schemas.SearchEventOperator): + if not is_likable(op): + return value + if isinstance(value, list): + r = [] + for v in value: + r.append(values_for_operator(v, op)) + return r else: - _value = value - if _value is None: - return _value - if op == schemas.SearchEventOperator._starts_with: - _value = '^' + _value - elif op == schemas.SearchEventOperator._ends_with: - _value = _value + '$' - return _value + if value is None: + return value + if op == schemas.SearchEventOperator._starts_with: + return value + '%%' + elif op == schemas.SearchEventOperator._ends_with: + return '%%' + value + elif op == schemas.SearchEventOperator._contains: + return '%%' + value + '%%' + return value def is_valid_email(email): diff --git a/api/schemas.py b/api/schemas.py index 5ec835dd1..b432dd0ac 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -330,6 +330,36 @@ class SourcemapUploadPayloadSchema(BaseModel): urls: List[str] = Field(..., alias="URL") +class ErrorSource(str, Enum): + js_exception = "js_exception" + bugsnag = "bugsnag" + cloudwatch = "cloudwatch" + datadog = "datadog" + newrelic = "newrelic" + rollbar = "rollbar" + sentry = "sentry" + stackdriver = "stackdriver" + sumologic = "sumologic" + + +class EventType(str, Enum): + click = "CLICK" + input = "INPUT" + location = "LOCATION" + custom = "CUSTOM" + request = "REQUEST" + graphql = "GRAPHQL" + state_action = "STATEACTION" + error = "ERROR" + metadata = "METADATA" + click_ios = "CLICK_IOS" + input_ios = "INPUT_IOS" + view_ios = "VIEW_IOS" + custom_ios = "CUSTOM_IOS" + request_ios = "REQUEST_IOS" + error_ios = "ERROR_IOS" + + class SearchEventOperator(str, Enum): _is = "is" _is_any = "isAny" @@ -348,13 +378,19 @@ class PlatformType(str, Enum): desktop = "desktop" +class SearchEventOrder(str, Enum): + _then = "then" + _or = "or" + _and = "and" + + class _SessionSearchEventSchema(BaseModel): - custom: Optional[str] = Field(None) + custom: Optional[List[str]] = Field(None) key: Optional[str] = Field(None) value: Union[Optional[str], Optional[List[str]]] = Field(...) - type: str = Field(...) + type: EventType = Field(...) operator: SearchEventOperator = Field(...) - source: Optional[str] = Field(...) + source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) class _SessionSearchFilterSchema(_SessionSearchEventSchema): @@ -371,6 +407,10 @@ class SessionsSearchPayloadSchema(BaseModel): sort: str = Field(...) order: str = Field(default="DESC") platform: Optional[PlatformType] = Field(None) + events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then) + + class Config: + alias_generator = attribute_to_camel_case class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema):