diff --git a/api/chalicelib/core/product_analytics/events.py b/api/chalicelib/core/product_analytics/events.py index 6e1e2e2cc..4b626d4aa 100644 --- a/api/chalicelib/core/product_analytics/events.py +++ b/api/chalicelib/core/product_analytics/events.py @@ -1,6 +1,12 @@ +import logging + +import schemas from chalicelib.utils import helper +from chalicelib.utils import sql_helper as sh from chalicelib.utils.ch_client import ClickHouseClient +logger = logging.getLogger(__name__) + def get_events(project_id: int): with ClickHouseClient() as ch_client: @@ -15,14 +21,88 @@ def get_events(project_id: int): return helper.list_to_camel_case(x) -def search_events(project_id: int, data: dict): +def search_events(project_id: int, data: schemas.EventsSearchPayloadSchema): with ClickHouseClient() as ch_client: - r = ch_client.format( - """SELECT * - FROM product_analytics.events - WHERE project_id=%(project_id)s - ORDER BY created_at;""", - parameters={"project_id": project_id}) - x = ch_client.execute(r) + full_args = {"project_id": project_id, "startDate": data.startTimestamp, "endDate": data.endTimestamp, + "projectId": project_id, "limit": data.limit, "offset": (data.page - 1) * data.limit} - return helper.list_to_camel_case(x) + constraints = ["project_id = %(projectId)s", + "created_at >= toDateTime(%(startDate)s/1000)", + "created_at <= toDateTime(%(endDate)s/1000)"] + for i, f in enumerate(data.filters): + f.value = helper.values_for_operator(value=f.value, op=f.operator) + f_k = f"f_value{i}" + full_args = {**full_args, f_k: sh.single_value(f.value), **sh.multi_values(f.value, value_key=f_k)} + op = sh.get_sql_operator(f.operator) + is_any = sh.isAny_opreator(f.operator) + is_undefined = sh.isUndefined_operator(f.operator) + full_args = {**full_args, f_k: sh.single_value(f.value), **sh.multi_values(f.value, value_key=f_k)} + if f.is_predefined: + column = f.name + else: + column = f"properties.{f.name}" + + if is_any: + condition = f"isNotNull({column})" + elif is_undefined: + condition = f"isNull({column})" + else: + condition = sh.multi_conditions(f"{column} {op} %({f_k})s", f.value, value_key=f_k) + constraints.append(condition) + + ev_constraints = [] + for i, e in enumerate(data.events): + e_k = f"e_value{i}" + full_args = {**full_args, e_k: e.event_name} + condition = f"`$event_name` = %({e_k})s" + sub_conditions = [] + if len(e.properties.filters) > 0: + for j, f in enumerate(e.properties.filters): + p_k = f"e_{i}_p_{j}" + full_args = {**full_args, **sh.multi_values(f.value, value_key=p_k)} + if f.is_predefined: + sub_condition = f"{f.name} {op} %({p_k})s" + else: + sub_condition = f"properties.{f.name} {op} %({p_k})s" + sub_conditions.append(sh.multi_conditions(sub_condition, f.value, value_key=p_k)) + if len(sub_conditions) > 0: + condition += " AND (" + for j, c in enumerate(sub_conditions): + if j > 0: + condition += " " + e.properties.operators[j - 1] + " " + c + else: + condition += c + condition += ")" + + ev_constraints.append(condition) + + constraints.append("(" + " OR ".join(ev_constraints) + ")") + query = ch_client.format( + f"""SELECT COUNT(1) OVER () AS total, + event_id, + `$event_name`, + created_at, + `distinct_id`, + `$browser`, + `$import`, + `$os`, + `$country`, + `$state`, + `$city`, + `$screen_height`, + `$screen_width`, + `$source`, + `$user_id`, + `$device` + FROM product_analytics.events + WHERE {" AND ".join(constraints)} + ORDER BY created_at + LIMIT %(limit)s OFFSET %(offset)s;""", + parameters=full_args) + rows = ch_client.execute(query) + if len(rows) == 0: + return {"total": 0, "rows": [], "src": 2} + total = rows[0]["total"] + for r in rows: + r.pop("total") + return {"total": total, "rows": rows, "src": 2} diff --git a/api/routers/subs/product_analytics.py b/api/routers/subs/product_analytics.py index 5bfe42de7..d8591b30d 100644 --- a/api/routers/subs/product_analytics.py +++ b/api/routers/subs/product_analytics.py @@ -3,6 +3,7 @@ from chalicelib.core.product_analytics import events, properties from fastapi import Depends from or_dependencies import OR_context from routers.base import get_routers +from fastapi import Body, Depends, BackgroundTasks public_app, app, app_apikey = get_routers() @@ -15,14 +16,13 @@ def get_event_properties(projectId: int, event_name: str = None, return {"data": properties.get_properties(project_id=projectId, event_name=event_name)} -@app.get('/{projectId}/events/names', tags=["dashboard"]) +@app.get('/{projectId}/events/names', tags=["product_analytics"]) def get_all_events(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": events.get_events(project_id=projectId)} -@app.post('/{projectId}/events/search', tags=["dashboard"]) -def search_events(projectId: int, - # data: schemas.CreateDashboardSchema = Body(...), +@app.post('/{projectId}/events/search', tags=["product_analytics"]) +def search_events(projectId: int, data: schemas.EventsSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": events.search_events(project_id=projectId, data={})} + return {"data": events.search_events(project_id=projectId, data=data)} diff --git a/api/schemas/__init__.py b/api/schemas/__init__.py index 6013d7c2b..4bcc5be80 100644 --- a/api/schemas/__init__.py +++ b/api/schemas/__init__.py @@ -1,2 +1,3 @@ from .schemas import * +from .product_analytics import * from . import overrides as _overrides diff --git a/api/schemas/product_analytics.py b/api/schemas/product_analytics.py new file mode 100644 index 000000000..bd4647b4e --- /dev/null +++ b/api/schemas/product_analytics.py @@ -0,0 +1,19 @@ +from typing import Optional, List + +from pydantic import Field + +from .overrides import BaseModel +from .schemas import EventPropertiesSchema, SortOrderType, _TimedSchema, \ + _PaginatedSchema, PropertyFilterSchema + + +class EventSearchSchema(BaseModel): + event_name: str = Field(...) + properties: Optional[EventPropertiesSchema] = Field(default=None) + + +class EventsSearchPayloadSchema(_TimedSchema, _PaginatedSchema): + events: List[EventSearchSchema] = Field(default_factory=list, description="operator between events is OR") + filters: List[PropertyFilterSchema] = Field(default_factory=list, description="operator between filters is AND") + sort: str = Field(default="startTs") + order: SortOrderType = Field(default=SortOrderType.DESC) diff --git a/ee/api/.gitignore b/ee/api/.gitignore index d46a28ff0..9e77b5867 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -292,4 +292,5 @@ Pipfile.lock /chalicelib/core/errors/errors_pg.py /chalicelib/core/errors/errors_ch.py /chalicelib/core/errors/errors_details.py -/chalicelib/utils/contextual_validators.py \ No newline at end of file +/chalicelib/utils/contextual_validators.py +/routers/subs/product_analytics.py diff --git a/ee/api/app.py b/ee/api/app.py index e1dece2dd..5b3af9d80 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -21,7 +21,7 @@ from chalicelib.utils import pg_client, ch_client from crons import core_crons, ee_crons, core_dynamic_crons from routers import core, core_dynamic from routers import ee -from routers.subs import insights, metrics, v1_api, health, usability_tests, spot, product_anaytics +from routers.subs import insights, metrics, v1_api, health, usability_tests, spot, product_analytics from routers.subs import v1_api_ee if config("ENABLE_SSO", cast=bool, default=True): @@ -150,9 +150,9 @@ app.include_router(spot.public_app) app.include_router(spot.app) app.include_router(spot.app_apikey) -app.include_router(product_anaytics.public_app, prefix="/ap") -app.include_router(product_anaytics.app, prefix="/ap") -app.include_router(product_anaytics.app_apikey, prefix="/ap") +app.include_router(product_analytics.public_app, prefix="/ap") +app.include_router(product_analytics.app, prefix="/ap") +app.include_router(product_analytics.app_apikey, prefix="/ap") if config("ENABLE_SSO", cast=bool, default=True): app.include_router(saml.public_app) diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index abfa59369..9bf06d6a0 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -113,3 +113,4 @@ rm -rf ./chalicelib/core/errors/errors_pg.py rm -rf ./chalicelib/core/errors/errors_ch.py rm -rf ./chalicelib/core/errors/errors_details.py rm -rf ./chalicelib/utils/contextual_validators.py +rm -rf ./routers/subs/product_analytics.py \ No newline at end of file