From f01a98c6191eb8fe98427f3589d1ef7cb1973e0e Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Wed, 31 Jul 2024 14:34:06 +0200 Subject: [PATCH] Dev (#2446) * refactor(chalice): upgraded dependencies * refactor(chalice): upgraded dependencies feat(chalice): support heatmaps * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * refactor(chalice): upgraded dependencies * refactor(chalice): upgraded dependencies feat(chalice): support heatmaps * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * refactor(chalice): upgraded dependencies refactor(crons): upgraded dependencies refactor(alerts): upgraded dependencies * feat(chalice): get top 10 values for autocomplete CH * refactor(chalice): cleaned code refactor(chalice): upgraded dependencies refactor(alerts): upgraded dependencies refactor(crons): upgraded dependencies * feat(chalice): autocomplete return top 10 with stats * fix(chalice): fixed autocomplete top 10 meta-filters * feat(chalice): support spot for EE --- ee/api/.gitignore | 1 + ee/api/auth/auth_jwt.py | 128 ++++++++++++------ ee/api/chalicelib/core/users.py | 54 ++++---- ee/api/clean-dev.sh | 2 + .../db/init_dbs/postgresql/1.20.0/1.20.0.sql | 4 + .../db/init_dbs/postgresql/init_schema.sql | 39 +++--- 6 files changed, 144 insertions(+), 84 deletions(-) diff --git a/ee/api/.gitignore b/ee/api/.gitignore index d2dfac7d0..3df5026e6 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -273,3 +273,4 @@ Pipfile.lock /chalicelib/core/usability_testing/ /NOTES.md /chalicelib/core/db_request_handler.py +/routers/subs/spot.py diff --git a/ee/api/auth/auth_jwt.py b/ee/api/auth/auth_jwt.py index e4b879217..5bcfd303a 100644 --- a/ee/api/auth/auth_jwt.py +++ b/ee/api/auth/auth_jwt.py @@ -9,7 +9,7 @@ from starlette import status from starlette.exceptions import HTTPException import schemas -from chalicelib.core import authorizers, users +from chalicelib.core import authorizers, users, spot logger = logging.getLogger(__name__) @@ -31,7 +31,8 @@ def _get_current_auth_context(request: Request, jwt_payload: dict) -> schemas.Cu def _allow_access_to_endpoint(request: Request, current_context: schemas.CurrentContext) -> bool: return not current_context.service_account \ - or request.url.path not in ["/logout", "/api/logout", "/refresh", "/api/refresh"] + or request.url.path not in ["/logout", "/api/logout", "/refresh", "/api/refresh", + "/spot/logout", "/api/spot/logout", "/spot/refresh", "/api/spot/refresh"] class JWTAuth(HTTPBearer): @@ -40,43 +41,10 @@ class JWTAuth(HTTPBearer): async def __call__(self, request: Request) -> Optional[schemas.CurrentContext]: if request.url.path in ["/refresh", "/api/refresh"]: - if "refreshToken" not in request.cookies: - logger.warning("Missing refreshToken cookie.") - jwt_payload = None - else: - jwt_payload = authorizers.jwt_refresh_authorizer(scheme="Bearer", token=request.cookies["refreshToken"]) + return await self.__process_refresh_call(request) - if jwt_payload is None or jwt_payload.get("jti") is None: - logger.warning("Null refreshToken's payload, or null JTI.") - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, - detail="Invalid refresh-token or expired refresh-token.") - auth_exists = users.refresh_auth_exists(user_id=jwt_payload.get("userId", -1), - tenant_id=jwt_payload.get("tenantId", -1), - jwt_jti=jwt_payload["jti"]) - if not auth_exists: - logger.warning("refreshToken's user not found.") - logger.warning(jwt_payload) - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, - detail="Invalid refresh-token or expired refresh-token.") - - credentials: HTTPAuthorizationCredentials = await super(JWTAuth, self).__call__(request) - if credentials: - if not credentials.scheme == "Bearer": - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, - detail="Invalid authentication scheme.") - old_jwt_payload = authorizers.jwt_authorizer(scheme=credentials.scheme, token=credentials.credentials, - leeway=datetime.timedelta( - days=config("JWT_LEEWAY_DAYS", cast=int, default=3) - )) - if old_jwt_payload is None \ - or old_jwt_payload.get("userId") is None \ - or old_jwt_payload.get("userId") != jwt_payload.get("userId"): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid token or expired token.") - - ctx = _get_current_auth_context(request=request, jwt_payload=jwt_payload) - if not _allow_access_to_endpoint(request=request, current_context=ctx): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Unauthorized endpoint.") - return ctx + elif request.url.path in ["/spot/refresh", "/spot/api/refresh"]: + return await self.__process_spot_refresh_call(request) else: credentials: HTTPAuthorizationCredentials = await super(JWTAuth, self).__call__(request) @@ -110,3 +78,87 @@ class JWTAuth(HTTPBearer): logger.warning("Invalid authorization code.") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid authorization code.") + + async def __process_refresh_call(self, request: Request) -> schemas.CurrentContext: + if "refreshToken" not in request.cookies: + logger.warning("Missing refreshToken cookie.") + jwt_payload = None + else: + jwt_payload = authorizers.jwt_refresh_authorizer(scheme="Bearer", token=request.cookies["refreshToken"]) + + if jwt_payload is None or jwt_payload.get("jti") is None: + logger.warning("Null refreshToken's payload, or null JTI.") + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid refresh-token or expired refresh-token.") + auth_exists = users.refresh_auth_exists(user_id=jwt_payload.get("userId", -1), + tenant_id=jwt_payload.get("tenantId", -1), + jwt_jti=jwt_payload["jti"]) + if not auth_exists: + logger.warning("refreshToken's user not found.") + logger.warning(jwt_payload) + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid refresh-token or expired refresh-token.") + + credentials: HTTPAuthorizationCredentials = await super(JWTAuth, self).__call__(request) + if credentials: + if not credentials.scheme == "Bearer": + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid authentication scheme.") + old_jwt_payload = authorizers.jwt_authorizer(scheme=credentials.scheme, token=credentials.credentials, + leeway=datetime.timedelta( + days=config("JWT_LEEWAY_DAYS", cast=int, default=3) + )) + if old_jwt_payload is None \ + or old_jwt_payload.get("userId") is None \ + or old_jwt_payload.get("userId") != jwt_payload.get("userId"): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid token or expired token.") + + ctx = _get_current_auth_context(request=request, jwt_payload=jwt_payload) + if not _allow_access_to_endpoint(request=request, current_context=ctx): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Unauthorized endpoint.") + return ctx + logger.warning("Invalid authorization code (refresh logic).") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid authorization code for refresh.") + + async def __process_spot_refresh_call(self, request: Request) -> schemas.CurrentContext: + if "refreshToken" not in request.cookies: + logger.warning("Missing sopt-refreshToken cookie.") + jwt_payload = None + else: + jwt_payload = authorizers.jwt_refresh_authorizer(scheme="Bearer", token=request.cookies["refreshToken"]) + + if jwt_payload is None or jwt_payload.get("jti") is None: + logger.warning("Null spot-refreshToken's payload, or null JTI.") + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid spot-refresh-token or expired refresh-token.") + auth_exists = spot.refresh_auth_exists(user_id=jwt_payload.get("userId", -1), + jwt_jti=jwt_payload["jti"]) + if not auth_exists: + logger.warning("spot-refreshToken's user not found.") + logger.warning(jwt_payload) + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid spot-refresh-token or expired refresh-token.") + + credentials: HTTPAuthorizationCredentials = await super(JWTAuth, self).__call__(request) + if credentials: + if not credentials.scheme == "Bearer": + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid spot-authentication scheme.") + old_jwt_payload = authorizers.jwt_authorizer(scheme=credentials.scheme, token=credentials.credentials, + leeway=datetime.timedelta( + days=config("JWT_LEEWAY_DAYS", cast=int, default=3) + )) + if old_jwt_payload is None \ + or old_jwt_payload.get("userId") is None \ + or old_jwt_payload.get("userId") != jwt_payload.get("userId"): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid spot-token or expired token.") + + ctx = _get_current_auth_context(request=request, jwt_payload=jwt_payload) + if not _allow_access_to_endpoint(request=request, current_context=ctx): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Unauthorized endpoint.") + return ctx + + logger.warning("Invalid authorization code (spot-refresh logic).") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid authorization code for spot-refresh.") diff --git a/ee/api/chalicelib/core/users.py b/ee/api/chalicelib/core/users.py index 7f930a40b..70bd865ea 100644 --- a/ee/api/chalicelib/core/users.py +++ b/ee/api/chalicelib/core/users.py @@ -16,6 +16,7 @@ from chalicelib.utils import pg_client from chalicelib.utils.TimeUTC import TimeUTC logger = logging.getLogger(__name__) +AUDIENCE = "front:OpenReplay" def __generate_invitation_token(): @@ -109,8 +110,7 @@ def restore_member(tenant_id, user_id, email, invitation_token, admin, name, own cur.execute(query) result = cur.fetchone() result["created_at"] = TimeUTC.datetime_to_timestamp(result["created_at"]) - - return helper.dict_to_camel_case(result) + return helper.dict_to_camel_case(result) def generate_new_invitation(user_id): @@ -165,22 +165,20 @@ def update(tenant_id, user_id, changes, output=True): changes["role_id"] = changes.get("roleId", changes.get("role_id")) with pg_client.PostgresClient() as cur: if len(sub_query_users) > 0: - cur.execute( - cur.mogrify(f"""\ + query = cur.mogrify(f"""\ UPDATE public.users SET {" ,".join(sub_query_users)} WHERE users.user_id = %(user_id)s AND users.tenant_id = %(tenant_id)s;""", - {"tenant_id": tenant_id, "user_id": user_id, **changes}) - ) + {"tenant_id": tenant_id, "user_id": user_id, **changes}) + cur.execute(query) if len(sub_query_bauth) > 0: - cur.execute( - cur.mogrify(f"""\ + query = cur.mogrify(f"""\ UPDATE public.basic_authentication SET {" ,".join(sub_query_bauth)} WHERE basic_authentication.user_id = %(user_id)s;""", - {"tenant_id": tenant_id, "user_id": user_id, **changes}) - ) + {"tenant_id": tenant_id, "user_id": user_id, **changes}) + cur.execute(query) if not output: return None return get(user_id=user_id, tenant_id=tenant_id) @@ -488,14 +486,15 @@ def delete_member(user_id, tenant_id, id_to_delete): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify(f"""UPDATE public.users - SET deleted_at = timezone('utc'::text, now()), role_id=NULL, + SET deleted_at = timezone('utc'::text, now()), jwt_iat= NULL, jwt_refresh_jti= NULL, - jwt_refresh_iat= NULL + jwt_refresh_iat= NULL, + role_id=NULL WHERE user_id=%(user_id)s AND tenant_id=%(tenant_id)s;""", {"user_id": id_to_delete, "tenant_id": tenant_id})) cur.execute( - cur.mogrify(f"""UPDATE public.basic_authentication - SET password=NULL, invitation_token= NULL, + cur.mogrify(f"""UPDATE public.basic_authentication + SET password= NULL, invitation_token= NULL, invited_at= NULL, changed_at= NULL, change_pwd_expire_at= NULL, change_pwd_token= NULL WHERE user_id=%(user_id)s;""", @@ -546,7 +545,7 @@ def set_password_invitation(tenant_id, user_id, new_password): 'jwt': r.pop('jwt'), 'data': { "user": r, - "client": c, + "client": c } } @@ -668,7 +667,7 @@ def refresh_jwt_iat_jti(user_id): WHERE user_id = %(user_id)s RETURNING EXTRACT (epoch FROM jwt_iat)::BIGINT AS jwt_iat, jwt_refresh_jti, - EXTRACT (epoch FROM jwt_refresh_iat)::BIGINT AS jwt_refresh_iat""", + EXTRACT (epoch FROM jwt_refresh_iat)::BIGINT AS jwt_refresh_iat;""", {"user_id": user_id}) cur.execute(query) row = cur.fetchone() @@ -729,9 +728,9 @@ def authenticate(email, password, for_change_password=False) -> dict | bool | No return { "jwt": authorizers.generate_jwt(user_id=r['userId'], tenant_id=r['tenantId'], iat=jwt_iat, - aud=f"front:{helper.get_stage_name()}"), + aud=AUDIENCE), "refreshToken": authorizers.generate_jwt_refresh(user_id=r['userId'], tenant_id=r['tenantId'], - iat=jwt_r_iat, aud=f"front:{helper.get_stage_name()}", + iat=jwt_r_iat, aud=AUDIENCE, jwt_jti=jwt_r_jti), "refreshTokenMaxAge": config("JWT_REFRESH_EXPIRATION", cast=int), "email": email, @@ -811,7 +810,8 @@ def logout(user_id: int): with pg_client.PostgresClient() as cur: query = cur.mogrify( """UPDATE public.users - SET jwt_iat = NULL, jwt_refresh_jti = NULL, jwt_refresh_iat = NULL + SET jwt_iat = NULL, jwt_refresh_jti = NULL, jwt_refresh_iat = NULL, + spot_jwt_iat = NULL, spot_jwt_refresh_jti = NULL, spot_jwt_refresh_iat = NULL WHERE user_id = %(user_id)s;""", {"user_id": user_id}) cur.execute(query) @@ -821,10 +821,9 @@ def refresh(user_id: int, tenant_id: int) -> dict: jwt_iat, jwt_r_jti, jwt_r_iat = refresh_jwt_iat_jti(user_id=user_id) return { "jwt": authorizers.generate_jwt(user_id=user_id, tenant_id=tenant_id, iat=jwt_iat, - aud=f"front:{helper.get_stage_name()}"), - "refreshToken": authorizers.generate_jwt_refresh(user_id=user_id, tenant_id=tenant_id, - iat=jwt_r_iat, aud=f"front:{helper.get_stage_name()}", - jwt_jti=jwt_r_jti), + aud=AUDIENCE), + "refreshToken": authorizers.generate_jwt_refresh(user_id=user_id, tenant_id=tenant_id, iat=jwt_r_iat, + aud=AUDIENCE, jwt_jti=jwt_r_jti), "refreshTokenMaxAge": config("JWT_REFRESH_EXPIRATION", cast=int) - (jwt_iat - jwt_r_iat) } @@ -858,10 +857,10 @@ def authenticate_sso(email, internal_id, exp=None): jwt_iat, jwt_r_jti, jwt_r_iat = change_jwt_iat_jti(user_id=r['userId']) return { "jwt": authorizers.generate_jwt(user_id=r['userId'], tenant_id=r['tenantId'], iat=jwt_iat, - aud=f"front:{helper.get_stage_name()}"), + aud=AUDIENCE), "refreshToken": authorizers.generate_jwt_refresh(user_id=r['userId'], tenant_id=r['tenantId'], - iat=jwt_r_iat, aud=f"front:{helper.get_stage_name()}", - jwt_jti=jwt_r_jti), + iat=jwt_r_iat, + aud=AUDIENCE, jwt_jti=jwt_r_jti), "refreshTokenMaxAge": config("JWT_REFRESH_EXPIRATION", cast=int), } logger.warning(f"SSO user not found with email: {email} and internal_id: {internal_id}") @@ -933,8 +932,7 @@ def get_user_settings(user_id): LIMIT 1""", {"user_id": user_id}) ) - settings = cur.fetchone() - return helper.dict_to_camel_case(settings) + return helper.dict_to_camel_case(cur.fetchone()) def update_user_module(user_id, data: schemas.ModuleStatus): diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index fee160254..250eeffd4 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -93,3 +93,5 @@ rm -rf ./schemas/transformers_validators.py rm -rf ./orpy.py rm -rf ./chalicelib/core/usability_testing/ rm -rf ./chalicelib/core/db_request_handler.py +rm -rf ./chalicelib/core/db_request_handler.py +rm -rf ./routers/subs/spot.py \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.20.0/1.20.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.20.0/1.20.0.sql index c69a20549..233afdb7f 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.20.0/1.20.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.20.0/1.20.0.sql @@ -32,6 +32,10 @@ SET permissions = (SELECT array_agg(distinct e) FROM unnest(permissions || '{SPO WHERE NOT permissions @> '{SPOT_PUBLIC}' AND name ILIKE 'owner'; +ALTER TABLE IF EXISTS public.users + ADD COLUMN IF NOT EXISTS spot_jwt_iat timestamp without time zone NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS spot_jwt_refresh_jti integer NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS spot_jwt_refresh_iat timestamp without time zone NULL DEFAULT NULL; COMMIT; diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index b1f9afb74..6fdfbf29a 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -125,24 +125,27 @@ CREATE TYPE user_role AS ENUM ('owner','admin','member','service'); CREATE TABLE public.users ( - user_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - tenant_id integer NOT NULL REFERENCES public.tenants (tenant_id) ON DELETE CASCADE, - email text NOT NULL UNIQUE, - role user_role NOT NULL DEFAULT 'member', - name text NOT NULL, - created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), - deleted_at timestamp without time zone NULL DEFAULT NULL, - api_key text UNIQUE DEFAULT generate_api_key(20) NOT NULL, - jwt_iat timestamp without time zone NULL DEFAULT NULL, - jwt_refresh_jti integer NULL DEFAULT NULL, - jwt_refresh_iat timestamp without time zone NULL DEFAULT NULL, - data jsonb NOT NULL DEFAULT '{}'::jsonb, - weekly_report boolean NOT NULL DEFAULT TRUE, - settings jsonb DEFAULT NULL, - origin text NULL DEFAULT NULL, - role_id integer REFERENCES public.roles (role_id) ON DELETE SET NULL, - internal_id text NULL DEFAULT NULL, - service_account bool NOT NULL DEFAULT FALSE + user_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + tenant_id integer NOT NULL REFERENCES public.tenants (tenant_id) ON DELETE CASCADE, + email text NOT NULL UNIQUE, + role user_role NOT NULL DEFAULT 'member', + name text NOT NULL, + created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), + deleted_at timestamp without time zone NULL DEFAULT NULL, + api_key text UNIQUE DEFAULT generate_api_key(20) NOT NULL, + jwt_iat timestamp without time zone NULL DEFAULT NULL, + jwt_refresh_jti integer NULL DEFAULT NULL, + jwt_refresh_iat timestamp without time zone NULL DEFAULT NULL, + spot_jwt_iat timestamp without time zone NULL DEFAULT NULL, + spot_jwt_refresh_jti integer NULL DEFAULT NULL, + spot_jwt_refresh_iat timestamp without time zone NULL DEFAULT NULL, + data jsonb NOT NULL DEFAULT '{}'::jsonb, + weekly_report boolean NOT NULL DEFAULT TRUE, + settings jsonb DEFAULT NULL, + origin text NULL DEFAULT NULL, + role_id integer REFERENCES public.roles (role_id) ON DELETE SET NULL, + internal_id text NULL DEFAULT NULL, + service_account bool NOT NULL DEFAULT FALSE ); CREATE INDEX users_tenant_id_deleted_at_N_idx ON public.users (tenant_id) WHERE deleted_at ISNULL; CREATE INDEX users_name_gin_idx ON public.users USING GIN (name gin_trgm_ops);