From 852ce7b324e3fbd7a719f3558b4db99c03242b20 Mon Sep 17 00:00:00 2001 From: MauricioGarciaS <47052044+MauricioGarciaS@users.noreply.github.com> Date: Mon, 23 Oct 2023 17:21:05 +0200 Subject: [PATCH] fix(redshift-connector): Redshift connector update (#1538) * Updated sql events table * Updated and fixed redshift connector python --- ee/connectors/Dockerfile_fill | 13 - ee/connectors/db/models.py | 4 + ee/connectors/db/utils.py | 4 + ee/connectors/deploy/Dockerfile_redshift | 23 - ee/connectors/deploy/Dockerfile_redshift_base | 4 +- .../deploy/requirements_redshift.txt | 22 +- ee/connectors/handler.py | 8 +- ee/connectors/msgcodec/msgcodec.pyx | 806 +++++++++--------- ee/connectors/requirements-fill.txt | 13 - ee/connectors/sql/redshift_events.sql | 4 + ee/connectors/utils/worker.py | 2 +- 11 files changed, 433 insertions(+), 470 deletions(-) delete mode 100644 ee/connectors/Dockerfile_fill delete mode 100644 ee/connectors/deploy/Dockerfile_redshift delete mode 100644 ee/connectors/requirements-fill.txt diff --git a/ee/connectors/Dockerfile_fill b/ee/connectors/Dockerfile_fill deleted file mode 100644 index 33a877eba..000000000 --- a/ee/connectors/Dockerfile_fill +++ /dev/null @@ -1,13 +0,0 @@ -FROM amancevice/pandas:2.0.2-alpine - -WORKDIR app - -COPY requirements-fill.txt . -RUN apk add --no-cache --virtual .build-deps gcc g++ musl-dev postgresql-dev && \ - pip install -r requirements-fill.txt --no-cache-dir && \ - apk --purge del .build-deps - -COPY utils utils -COPY fill_from_db.py . -COPY entrypoint-fill.sh . -ENTRYPOINT ./entrypoint-fill.sh diff --git a/ee/connectors/db/models.py b/ee/connectors/db/models.py index 7e80e1313..8bcc3171c 100644 --- a/ee/connectors/db/models.py +++ b/ee/connectors/db/models.py @@ -86,6 +86,10 @@ class Event(Base): consolelog_value = Column(VARCHAR(5000)) customevent_name = Column(VARCHAR(5000)) customevent_payload = Column(VARCHAR(5000)) + clickevent_hesitationtime = Column(BigInteger) + clickevent_messageid = Column(BigInteger) + clickevent_label = Column(VARCHAR(5000)) + clickevent_selector = Column(VARCHAR(5000)) jsexception_message = Column(VARCHAR(5000)) jsexception_name = Column(VARCHAR(5000)) jsexception_payload = Column(VARCHAR(5000)) diff --git a/ee/connectors/db/utils.py b/ee/connectors/db/utils.py index 3a7c9c53a..ed66f1a71 100644 --- a/ee/connectors/db/utils.py +++ b/ee/connectors/db/utils.py @@ -5,6 +5,10 @@ dtypes_events = { 'sessionid': "Int64", 'consolelog_level': "string", 'consolelog_value': "string", + 'clickevent_hesitationtime': "Int64", + 'clickevent_label': "string", + 'clickevent_messageid': "Int64", + 'clickevent_selector': "string", 'customevent_name': "string", 'customevent_payload': "string", 'jsexception_message': "string", diff --git a/ee/connectors/deploy/Dockerfile_redshift b/ee/connectors/deploy/Dockerfile_redshift deleted file mode 100644 index 75af55b15..000000000 --- a/ee/connectors/deploy/Dockerfile_redshift +++ /dev/null @@ -1,23 +0,0 @@ -FROM public.ecr.aws/p1t3u8a3/connectors/redshift:base - -ENV CLOUD_SERVICE=redshift \ - CONNECTION_STRING=postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{DBNAME} \ - # Keep postgres connection - PG_MINCONN=3 \ - PG_MAXCONN=10 -RUN apk add --no-cache postgresql-libs lz4-libs zstd-libs -COPY deploy/requirements_redshift.txt . -COPY msgcodec msgcodec -COPY build_modules.sh . -RUN apk add --no-cache --virtual .build-deps gcc g++ musl-dev postgresql-dev && \ - ./build_modules.sh && python3 -m pip install -r requirements_redshift.txt --no-cache-dir && \ - apk --purge del .build-deps -COPY utils utils -COPY db db -COPY sql sql -COPY handler.py . -COPY consumer_pool.py . -COPY fill_from_db.py . -COPY entrypoint.sh . -ENV replace_interval=300 -ENTRYPOINT ./entrypoint.sh diff --git a/ee/connectors/deploy/Dockerfile_redshift_base b/ee/connectors/deploy/Dockerfile_redshift_base index 2fc81d7f0..158e1c695 100644 --- a/ee/connectors/deploy/Dockerfile_redshift_base +++ b/ee/connectors/deploy/Dockerfile_redshift_base @@ -1,7 +1,7 @@ -FROM amancevice/pandas:2.0.2-alpine +FROM amancevice/pandas:alpine-2.1.1 WORKDIR /usr/src/app -ENV LIBRD_VER=2.1.1 +ENV LIBRD_VER=2.2.0 WORKDIR /work RUN apk add --no-cache --virtual .make-deps postgresql-dev gcc python3-dev \ musl-dev linux-headers g++ libc-dev libffi-dev make cmake py-pip build-base \ diff --git a/ee/connectors/deploy/requirements_redshift.txt b/ee/connectors/deploy/requirements_redshift.txt index 51696ca24..75caa1098 100644 --- a/ee/connectors/deploy/requirements_redshift.txt +++ b/ee/connectors/deploy/requirements_redshift.txt @@ -1,15 +1,15 @@ -chardet==5.1.0 +chardet==5.2.0 idna==3.4 -confluent-kafka==2.1.1 -psycopg2-binary==2.9.6 -apscheduler==3.10.1 +confluent-kafka==2.2.0 +psycopg2-binary==2.9.9 +apscheduler==3.10.4 python-decouple==3.8 -pytz==2022.6 -requests==2.28.1 -SQLAlchemy==1.4.48 -tzlocal==5.0.1 -urllib3==1.26.15 +pytz==2023.3.post1 +requests==2.31.0 +SQLAlchemy==1.4.49 +tzlocal==5.2 +urllib3==2.0.7 sqlalchemy-redshift==0.8.14 -redshift-connector==2.0.911 +redshift-connector==2.0.915 pandas-redshift==2.0.5 -PyYAML==6.0 +PyYAML==6.0.1 diff --git a/ee/connectors/handler.py b/ee/connectors/handler.py index 40f601eaf..3a7d0d27b 100644 --- a/ee/connectors/handler.py +++ b/ee/connectors/handler.py @@ -29,10 +29,10 @@ def handle_normal_message(message: Message) -> Optional[Event]: return n if isinstance(message, MouseClick): - n.mouseclick_hesitationtime = message.hesitation_time - n.mouseclick_id = message.id - n.mouseclick_label = message.label - n.mouseclick_selector = message.selector + n.clickevent_hesitationtime = message.hesitation_time + n.clickevent_messageid = message.id + n.clickevent_label = message.label + n.clickevent_selector = message.selector return n if isinstance(message, NetworkRequest): diff --git a/ee/connectors/msgcodec/msgcodec.pyx b/ee/connectors/msgcodec/msgcodec.pyx index dc0aa4ced..003273a30 100644 --- a/ee/connectors/msgcodec/msgcodec.pyx +++ b/ee/connectors/msgcodec/msgcodec.pyx @@ -173,18 +173,18 @@ cdef class MessageCodec: return messages_list def handler(self, PyBytesIO reader, int mode = 0): - cdef unsigned long message_id = MessageCodec.read_message_id(reader) + cdef unsigned long message_id = self.read_message_id(reader) cdef int r_size if mode == 1: # We read the three bytes representing the length of message. It can be used to skip unwanted messages - r_size = MessageCodec.read_size(reader) + r_size = self.read_size(reader) if message_id not in self.msg_selector: reader.read(r_size) return None - return MessageCodec.read_head_message(reader, message_id) + return self.read_head_message(reader, message_id) elif mode == 0: # Old format with no bytes for message length - return MessageCodec.read_head_message(reader, message_id) + return self.read_head_message(reader, message_id) else: raise IOError() @@ -193,51 +193,51 @@ cdef class MessageCodec: if message_id == 0: return Timestamp( - timestamp=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader) ) if message_id == 1: return SessionStart( - timestamp=self.read_uint(reader), - project_id=self.read_uint(reader), - tracker_version=self.read_string(reader), - rev_id=self.read_string(reader), - user_uuid=self.read_string(reader), - user_agent=self.read_string(reader), - user_os=self.read_string(reader), - user_os_version=self.read_string(reader), - user_browser=self.read_string(reader), - user_browser_version=self.read_string(reader), - user_device=self.read_string(reader), - user_device_type=self.read_string(reader), - user_device_memory_size=self.read_uint(reader), - user_device_heap_size=self.read_uint(reader), - user_country=self.read_string(reader), - user_id=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + project_id=MessageCodec.read_uint(reader), + tracker_version=MessageCodec.read_string(reader), + rev_id=MessageCodec.read_string(reader), + user_uuid=MessageCodec.read_string(reader), + user_agent=MessageCodec.read_string(reader), + user_os=MessageCodec.read_string(reader), + user_os_version=MessageCodec.read_string(reader), + user_browser=MessageCodec.read_string(reader), + user_browser_version=MessageCodec.read_string(reader), + user_device=MessageCodec.read_string(reader), + user_device_type=MessageCodec.read_string(reader), + user_device_memory_size=MessageCodec.read_uint(reader), + user_device_heap_size=MessageCodec.read_uint(reader), + user_country=MessageCodec.read_string(reader), + user_id=MessageCodec.read_string(reader) ) if message_id == 3: return SessionEndDeprecated( - timestamp=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader) ) if message_id == 4: return SetPageLocation( - url=self.read_string(reader), - referrer=self.read_string(reader), - navigation_start=self.read_uint(reader) + url=MessageCodec.read_string(reader), + referrer=MessageCodec.read_string(reader), + navigation_start=MessageCodec.read_uint(reader) ) if message_id == 5: return SetViewportSize( - width=self.read_uint(reader), - height=self.read_uint(reader) + width=MessageCodec.read_uint(reader), + height=MessageCodec.read_uint(reader) ) if message_id == 6: return SetViewportScroll( - x=self.read_int(reader), - y=self.read_int(reader) + x=MessageCodec.read_int(reader), + y=MessageCodec.read_int(reader) ) if message_id == 7: @@ -247,763 +247,763 @@ cdef class MessageCodec: if message_id == 8: return CreateElementNode( - id=self.read_uint(reader), - parent_id=self.read_uint(reader), - index=self.read_uint(reader), - tag=self.read_string(reader), - svg=self.read_boolean(reader) + id=MessageCodec.read_uint(reader), + parent_id=MessageCodec.read_uint(reader), + index=MessageCodec.read_uint(reader), + tag=MessageCodec.read_string(reader), + svg=MessageCodec.read_boolean(reader) ) if message_id == 9: return CreateTextNode( - id=self.read_uint(reader), - parent_id=self.read_uint(reader), - index=self.read_uint(reader) + id=MessageCodec.read_uint(reader), + parent_id=MessageCodec.read_uint(reader), + index=MessageCodec.read_uint(reader) ) if message_id == 10: return MoveNode( - id=self.read_uint(reader), - parent_id=self.read_uint(reader), - index=self.read_uint(reader) + id=MessageCodec.read_uint(reader), + parent_id=MessageCodec.read_uint(reader), + index=MessageCodec.read_uint(reader) ) if message_id == 11: return RemoveNode( - id=self.read_uint(reader) + id=MessageCodec.read_uint(reader) ) if message_id == 12: return SetNodeAttribute( - id=self.read_uint(reader), - name=self.read_string(reader), - value=self.read_string(reader) + id=MessageCodec.read_uint(reader), + name=MessageCodec.read_string(reader), + value=MessageCodec.read_string(reader) ) if message_id == 13: return RemoveNodeAttribute( - id=self.read_uint(reader), - name=self.read_string(reader) + id=MessageCodec.read_uint(reader), + name=MessageCodec.read_string(reader) ) if message_id == 14: return SetNodeData( - id=self.read_uint(reader), - data=self.read_string(reader) + id=MessageCodec.read_uint(reader), + data=MessageCodec.read_string(reader) ) if message_id == 15: return SetCSSData( - id=self.read_uint(reader), - data=self.read_string(reader) + id=MessageCodec.read_uint(reader), + data=MessageCodec.read_string(reader) ) if message_id == 16: return SetNodeScroll( - id=self.read_uint(reader), - x=self.read_int(reader), - y=self.read_int(reader) + id=MessageCodec.read_uint(reader), + x=MessageCodec.read_int(reader), + y=MessageCodec.read_int(reader) ) if message_id == 17: return SetInputTarget( - id=self.read_uint(reader), - label=self.read_string(reader) + id=MessageCodec.read_uint(reader), + label=MessageCodec.read_string(reader) ) if message_id == 18: return SetInputValue( - id=self.read_uint(reader), - value=self.read_string(reader), - mask=self.read_int(reader) + id=MessageCodec.read_uint(reader), + value=MessageCodec.read_string(reader), + mask=MessageCodec.read_int(reader) ) if message_id == 19: return SetInputChecked( - id=self.read_uint(reader), - checked=self.read_boolean(reader) + id=MessageCodec.read_uint(reader), + checked=MessageCodec.read_boolean(reader) ) if message_id == 20: return MouseMove( - x=self.read_uint(reader), - y=self.read_uint(reader) + x=MessageCodec.read_uint(reader), + y=MessageCodec.read_uint(reader) ) if message_id == 21: return NetworkRequestDeprecated( - type=self.read_string(reader), - method=self.read_string(reader), - url=self.read_string(reader), - request=self.read_string(reader), - response=self.read_string(reader), - status=self.read_uint(reader), - timestamp=self.read_uint(reader), - duration=self.read_uint(reader) + type=MessageCodec.read_string(reader), + method=MessageCodec.read_string(reader), + url=MessageCodec.read_string(reader), + request=MessageCodec.read_string(reader), + response=MessageCodec.read_string(reader), + status=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_uint(reader), + duration=MessageCodec.read_uint(reader) ) if message_id == 22: return ConsoleLog( - level=self.read_string(reader), - value=self.read_string(reader) + level=MessageCodec.read_string(reader), + value=MessageCodec.read_string(reader) ) if message_id == 23: return PageLoadTiming( - request_start=self.read_uint(reader), - response_start=self.read_uint(reader), - response_end=self.read_uint(reader), - dom_content_loaded_event_start=self.read_uint(reader), - dom_content_loaded_event_end=self.read_uint(reader), - load_event_start=self.read_uint(reader), - load_event_end=self.read_uint(reader), - first_paint=self.read_uint(reader), - first_contentful_paint=self.read_uint(reader) + request_start=MessageCodec.read_uint(reader), + response_start=MessageCodec.read_uint(reader), + response_end=MessageCodec.read_uint(reader), + dom_content_loaded_event_start=MessageCodec.read_uint(reader), + dom_content_loaded_event_end=MessageCodec.read_uint(reader), + load_event_start=MessageCodec.read_uint(reader), + load_event_end=MessageCodec.read_uint(reader), + first_paint=MessageCodec.read_uint(reader), + first_contentful_paint=MessageCodec.read_uint(reader) ) if message_id == 24: return PageRenderTiming( - speed_index=self.read_uint(reader), - visually_complete=self.read_uint(reader), - time_to_interactive=self.read_uint(reader) + speed_index=MessageCodec.read_uint(reader), + visually_complete=MessageCodec.read_uint(reader), + time_to_interactive=MessageCodec.read_uint(reader) ) if message_id == 25: return JSExceptionDeprecated( - name=self.read_string(reader), - message=self.read_string(reader), - payload=self.read_string(reader) + name=MessageCodec.read_string(reader), + message=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader) ) if message_id == 26: return IntegrationEvent( - timestamp=self.read_uint(reader), - source=self.read_string(reader), - name=self.read_string(reader), - message=self.read_string(reader), - payload=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + source=MessageCodec.read_string(reader), + name=MessageCodec.read_string(reader), + message=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader) ) if message_id == 27: return CustomEvent( - name=self.read_string(reader), - payload=self.read_string(reader) + name=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader) ) if message_id == 28: return UserID( - id=self.read_string(reader) + id=MessageCodec.read_string(reader) ) if message_id == 29: return UserAnonymousID( - id=self.read_string(reader) + id=MessageCodec.read_string(reader) ) if message_id == 30: return Metadata( - key=self.read_string(reader), - value=self.read_string(reader) + key=MessageCodec.read_string(reader), + value=MessageCodec.read_string(reader) ) if message_id == 31: return PageEvent( - message_id=self.read_uint(reader), - timestamp=self.read_uint(reader), - url=self.read_string(reader), - referrer=self.read_string(reader), - loaded=self.read_boolean(reader), - request_start=self.read_uint(reader), - response_start=self.read_uint(reader), - response_end=self.read_uint(reader), - dom_content_loaded_event_start=self.read_uint(reader), - dom_content_loaded_event_end=self.read_uint(reader), - load_event_start=self.read_uint(reader), - load_event_end=self.read_uint(reader), - first_paint=self.read_uint(reader), - first_contentful_paint=self.read_uint(reader), - speed_index=self.read_uint(reader), - visually_complete=self.read_uint(reader), - time_to_interactive=self.read_uint(reader) + message_id=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_uint(reader), + url=MessageCodec.read_string(reader), + referrer=MessageCodec.read_string(reader), + loaded=MessageCodec.read_boolean(reader), + request_start=MessageCodec.read_uint(reader), + response_start=MessageCodec.read_uint(reader), + response_end=MessageCodec.read_uint(reader), + dom_content_loaded_event_start=MessageCodec.read_uint(reader), + dom_content_loaded_event_end=MessageCodec.read_uint(reader), + load_event_start=MessageCodec.read_uint(reader), + load_event_end=MessageCodec.read_uint(reader), + first_paint=MessageCodec.read_uint(reader), + first_contentful_paint=MessageCodec.read_uint(reader), + speed_index=MessageCodec.read_uint(reader), + visually_complete=MessageCodec.read_uint(reader), + time_to_interactive=MessageCodec.read_uint(reader) ) if message_id == 32: return InputEvent( - message_id=self.read_uint(reader), - timestamp=self.read_uint(reader), - value=self.read_string(reader), - value_masked=self.read_boolean(reader), - label=self.read_string(reader) + message_id=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_uint(reader), + value=MessageCodec.read_string(reader), + value_masked=MessageCodec.read_boolean(reader), + label=MessageCodec.read_string(reader) ) if message_id == 37: return CSSInsertRule( - id=self.read_uint(reader), - rule=self.read_string(reader), - index=self.read_uint(reader) + id=MessageCodec.read_uint(reader), + rule=MessageCodec.read_string(reader), + index=MessageCodec.read_uint(reader) ) if message_id == 38: return CSSDeleteRule( - id=self.read_uint(reader), - index=self.read_uint(reader) + id=MessageCodec.read_uint(reader), + index=MessageCodec.read_uint(reader) ) if message_id == 39: return Fetch( - method=self.read_string(reader), - url=self.read_string(reader), - request=self.read_string(reader), - response=self.read_string(reader), - status=self.read_uint(reader), - timestamp=self.read_uint(reader), - duration=self.read_uint(reader) + method=MessageCodec.read_string(reader), + url=MessageCodec.read_string(reader), + request=MessageCodec.read_string(reader), + response=MessageCodec.read_string(reader), + status=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_uint(reader), + duration=MessageCodec.read_uint(reader) ) if message_id == 40: return Profiler( - name=self.read_string(reader), - duration=self.read_uint(reader), - args=self.read_string(reader), - result=self.read_string(reader) + name=MessageCodec.read_string(reader), + duration=MessageCodec.read_uint(reader), + args=MessageCodec.read_string(reader), + result=MessageCodec.read_string(reader) ) if message_id == 41: return OTable( - key=self.read_string(reader), - value=self.read_string(reader) + key=MessageCodec.read_string(reader), + value=MessageCodec.read_string(reader) ) if message_id == 42: return StateAction( - type=self.read_string(reader) + type=MessageCodec.read_string(reader) ) if message_id == 44: return Redux( - action=self.read_string(reader), - state=self.read_string(reader), - duration=self.read_uint(reader) + action=MessageCodec.read_string(reader), + state=MessageCodec.read_string(reader), + duration=MessageCodec.read_uint(reader) ) if message_id == 45: return Vuex( - mutation=self.read_string(reader), - state=self.read_string(reader) + mutation=MessageCodec.read_string(reader), + state=MessageCodec.read_string(reader) ) if message_id == 46: return MobX( - type=self.read_string(reader), - payload=self.read_string(reader) + type=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader) ) if message_id == 47: return NgRx( - action=self.read_string(reader), - state=self.read_string(reader), - duration=self.read_uint(reader) + action=MessageCodec.read_string(reader), + state=MessageCodec.read_string(reader), + duration=MessageCodec.read_uint(reader) ) if message_id == 48: return GraphQL( - operation_kind=self.read_string(reader), - operation_name=self.read_string(reader), - variables=self.read_string(reader), - response=self.read_string(reader) + operation_kind=MessageCodec.read_string(reader), + operation_name=MessageCodec.read_string(reader), + variables=MessageCodec.read_string(reader), + response=MessageCodec.read_string(reader) ) if message_id == 49: return PerformanceTrack( - frames=self.read_int(reader), - ticks=self.read_int(reader), - total_js_heap_size=self.read_uint(reader), - used_js_heap_size=self.read_uint(reader) + frames=MessageCodec.read_int(reader), + ticks=MessageCodec.read_int(reader), + total_js_heap_size=MessageCodec.read_uint(reader), + used_js_heap_size=MessageCodec.read_uint(reader) ) if message_id == 50: return StringDict( - key=self.read_uint(reader), - value=self.read_string(reader) + key=MessageCodec.read_uint(reader), + value=MessageCodec.read_string(reader) ) if message_id == 51: return SetNodeAttributeDict( - id=self.read_uint(reader), - name_key=self.read_uint(reader), - value_key=self.read_uint(reader) + id=MessageCodec.read_uint(reader), + name_key=MessageCodec.read_uint(reader), + value_key=MessageCodec.read_uint(reader) ) if message_id == 53: return ResourceTimingDeprecated( - timestamp=self.read_uint(reader), - duration=self.read_uint(reader), - ttfb=self.read_uint(reader), - header_size=self.read_uint(reader), - encoded_body_size=self.read_uint(reader), - decoded_body_size=self.read_uint(reader), - url=self.read_string(reader), - initiator=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + duration=MessageCodec.read_uint(reader), + ttfb=MessageCodec.read_uint(reader), + header_size=MessageCodec.read_uint(reader), + encoded_body_size=MessageCodec.read_uint(reader), + decoded_body_size=MessageCodec.read_uint(reader), + url=MessageCodec.read_string(reader), + initiator=MessageCodec.read_string(reader) ) if message_id == 54: return ConnectionInformation( - downlink=self.read_uint(reader), - type=self.read_string(reader) + downlink=MessageCodec.read_uint(reader), + type=MessageCodec.read_string(reader) ) if message_id == 55: return SetPageVisibility( - hidden=self.read_boolean(reader) + hidden=MessageCodec.read_boolean(reader) ) if message_id == 56: return PerformanceTrackAggr( - timestamp_start=self.read_uint(reader), - timestamp_end=self.read_uint(reader), - min_fps=self.read_uint(reader), - avg_fps=self.read_uint(reader), - max_fps=self.read_uint(reader), - min_cpu=self.read_uint(reader), - avg_cpu=self.read_uint(reader), - max_cpu=self.read_uint(reader), - min_total_js_heap_size=self.read_uint(reader), - avg_total_js_heap_size=self.read_uint(reader), - max_total_js_heap_size=self.read_uint(reader), - min_used_js_heap_size=self.read_uint(reader), - avg_used_js_heap_size=self.read_uint(reader), - max_used_js_heap_size=self.read_uint(reader) + timestamp_start=MessageCodec.read_uint(reader), + timestamp_end=MessageCodec.read_uint(reader), + min_fps=MessageCodec.read_uint(reader), + avg_fps=MessageCodec.read_uint(reader), + max_fps=MessageCodec.read_uint(reader), + min_cpu=MessageCodec.read_uint(reader), + avg_cpu=MessageCodec.read_uint(reader), + max_cpu=MessageCodec.read_uint(reader), + min_total_js_heap_size=MessageCodec.read_uint(reader), + avg_total_js_heap_size=MessageCodec.read_uint(reader), + max_total_js_heap_size=MessageCodec.read_uint(reader), + min_used_js_heap_size=MessageCodec.read_uint(reader), + avg_used_js_heap_size=MessageCodec.read_uint(reader), + max_used_js_heap_size=MessageCodec.read_uint(reader) ) if message_id == 57: return LoadFontFace( - parent_id=self.read_uint(reader), - family=self.read_string(reader), - source=self.read_string(reader), - descriptors=self.read_string(reader) + parent_id=MessageCodec.read_uint(reader), + family=MessageCodec.read_string(reader), + source=MessageCodec.read_string(reader), + descriptors=MessageCodec.read_string(reader) ) if message_id == 58: return SetNodeFocus( - id=self.read_int(reader) + id=MessageCodec.read_int(reader) ) if message_id == 59: return LongTask( - timestamp=self.read_uint(reader), - duration=self.read_uint(reader), - context=self.read_uint(reader), - container_type=self.read_uint(reader), - container_src=self.read_string(reader), - container_id=self.read_string(reader), - container_name=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + duration=MessageCodec.read_uint(reader), + context=MessageCodec.read_uint(reader), + container_type=MessageCodec.read_uint(reader), + container_src=MessageCodec.read_string(reader), + container_id=MessageCodec.read_string(reader), + container_name=MessageCodec.read_string(reader) ) if message_id == 60: return SetNodeAttributeURLBased( - id=self.read_uint(reader), - name=self.read_string(reader), - value=self.read_string(reader), - base_url=self.read_string(reader) + id=MessageCodec.read_uint(reader), + name=MessageCodec.read_string(reader), + value=MessageCodec.read_string(reader), + base_url=MessageCodec.read_string(reader) ) if message_id == 61: return SetCSSDataURLBased( - id=self.read_uint(reader), - data=self.read_string(reader), - base_url=self.read_string(reader) + id=MessageCodec.read_uint(reader), + data=MessageCodec.read_string(reader), + base_url=MessageCodec.read_string(reader) ) if message_id == 62: return IssueEventDeprecated( - message_id=self.read_uint(reader), - timestamp=self.read_uint(reader), - type=self.read_string(reader), - context_string=self.read_string(reader), - context=self.read_string(reader), - payload=self.read_string(reader) + message_id=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_uint(reader), + type=MessageCodec.read_string(reader), + context_string=MessageCodec.read_string(reader), + context=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader) ) if message_id == 63: return TechnicalInfo( - type=self.read_string(reader), - value=self.read_string(reader) + type=MessageCodec.read_string(reader), + value=MessageCodec.read_string(reader) ) if message_id == 64: return CustomIssue( - name=self.read_string(reader), - payload=self.read_string(reader) + name=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader) ) if message_id == 66: return AssetCache( - url=self.read_string(reader) + url=MessageCodec.read_string(reader) ) if message_id == 67: return CSSInsertRuleURLBased( - id=self.read_uint(reader), - rule=self.read_string(reader), - index=self.read_uint(reader), - base_url=self.read_string(reader) + id=MessageCodec.read_uint(reader), + rule=MessageCodec.read_string(reader), + index=MessageCodec.read_uint(reader), + base_url=MessageCodec.read_string(reader) ) if message_id == 69: return MouseClick( - id=self.read_uint(reader), - hesitation_time=self.read_uint(reader), - label=self.read_string(reader), - selector=self.read_string(reader) + id=MessageCodec.read_uint(reader), + hesitation_time=MessageCodec.read_uint(reader), + label=MessageCodec.read_string(reader), + selector=MessageCodec.read_string(reader) ) if message_id == 70: return CreateIFrameDocument( - frame_id=self.read_uint(reader), - id=self.read_uint(reader) + frame_id=MessageCodec.read_uint(reader), + id=MessageCodec.read_uint(reader) ) if message_id == 71: return AdoptedSSReplaceURLBased( - sheet_id=self.read_uint(reader), - text=self.read_string(reader), - base_url=self.read_string(reader) + sheet_id=MessageCodec.read_uint(reader), + text=MessageCodec.read_string(reader), + base_url=MessageCodec.read_string(reader) ) if message_id == 72: return AdoptedSSReplace( - sheet_id=self.read_uint(reader), - text=self.read_string(reader) + sheet_id=MessageCodec.read_uint(reader), + text=MessageCodec.read_string(reader) ) if message_id == 73: return AdoptedSSInsertRuleURLBased( - sheet_id=self.read_uint(reader), - rule=self.read_string(reader), - index=self.read_uint(reader), - base_url=self.read_string(reader) + sheet_id=MessageCodec.read_uint(reader), + rule=MessageCodec.read_string(reader), + index=MessageCodec.read_uint(reader), + base_url=MessageCodec.read_string(reader) ) if message_id == 74: return AdoptedSSInsertRule( - sheet_id=self.read_uint(reader), - rule=self.read_string(reader), - index=self.read_uint(reader) + sheet_id=MessageCodec.read_uint(reader), + rule=MessageCodec.read_string(reader), + index=MessageCodec.read_uint(reader) ) if message_id == 75: return AdoptedSSDeleteRule( - sheet_id=self.read_uint(reader), - index=self.read_uint(reader) + sheet_id=MessageCodec.read_uint(reader), + index=MessageCodec.read_uint(reader) ) if message_id == 76: return AdoptedSSAddOwner( - sheet_id=self.read_uint(reader), - id=self.read_uint(reader) + sheet_id=MessageCodec.read_uint(reader), + id=MessageCodec.read_uint(reader) ) if message_id == 77: return AdoptedSSRemoveOwner( - sheet_id=self.read_uint(reader), - id=self.read_uint(reader) + sheet_id=MessageCodec.read_uint(reader), + id=MessageCodec.read_uint(reader) ) if message_id == 78: return JSException( - name=self.read_string(reader), - message=self.read_string(reader), - payload=self.read_string(reader), - metadata=self.read_string(reader) + name=MessageCodec.read_string(reader), + message=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader), + metadata=MessageCodec.read_string(reader) ) if message_id == 79: return Zustand( - mutation=self.read_string(reader), - state=self.read_string(reader) + mutation=MessageCodec.read_string(reader), + state=MessageCodec.read_string(reader) ) if message_id == 80: return BatchMeta( - page_no=self.read_uint(reader), - first_index=self.read_uint(reader), - timestamp=self.read_int(reader) + page_no=MessageCodec.read_uint(reader), + first_index=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_int(reader) ) if message_id == 81: return BatchMetadata( - version=self.read_uint(reader), - page_no=self.read_uint(reader), - first_index=self.read_uint(reader), - timestamp=self.read_int(reader), - location=self.read_string(reader) + version=MessageCodec.read_uint(reader), + page_no=MessageCodec.read_uint(reader), + first_index=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_int(reader), + location=MessageCodec.read_string(reader) ) if message_id == 82: return PartitionedMessage( - part_no=self.read_uint(reader), - part_total=self.read_uint(reader) + part_no=MessageCodec.read_uint(reader), + part_total=MessageCodec.read_uint(reader) ) if message_id == 83: return NetworkRequest( - type=self.read_string(reader), - method=self.read_string(reader), - url=self.read_string(reader), - request=self.read_string(reader), - response=self.read_string(reader), - status=self.read_uint(reader), - timestamp=self.read_uint(reader), - duration=self.read_uint(reader), - transferred_body_size=self.read_uint(reader) + type=MessageCodec.read_string(reader), + method=MessageCodec.read_string(reader), + url=MessageCodec.read_string(reader), + request=MessageCodec.read_string(reader), + response=MessageCodec.read_string(reader), + status=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_uint(reader), + duration=MessageCodec.read_uint(reader), + transferred_body_size=MessageCodec.read_uint(reader) ) if message_id == 112: return InputChange( - id=self.read_uint(reader), - value=self.read_string(reader), - value_masked=self.read_boolean(reader), - label=self.read_string(reader), - hesitation_time=self.read_int(reader), - input_duration=self.read_int(reader) + id=MessageCodec.read_uint(reader), + value=MessageCodec.read_string(reader), + value_masked=MessageCodec.read_boolean(reader), + label=MessageCodec.read_string(reader), + hesitation_time=MessageCodec.read_int(reader), + input_duration=MessageCodec.read_int(reader) ) if message_id == 113: return SelectionChange( - selection_start=self.read_uint(reader), - selection_end=self.read_uint(reader), - selection=self.read_string(reader) + selection_start=MessageCodec.read_uint(reader), + selection_end=MessageCodec.read_uint(reader), + selection=MessageCodec.read_string(reader) ) if message_id == 114: return MouseThrashing( - timestamp=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader) ) if message_id == 115: return UnbindNodes( - total_removed_percent=self.read_uint(reader) + total_removed_percent=MessageCodec.read_uint(reader) ) if message_id == 116: return ResourceTiming( - timestamp=self.read_uint(reader), - duration=self.read_uint(reader), - ttfb=self.read_uint(reader), - header_size=self.read_uint(reader), - encoded_body_size=self.read_uint(reader), - decoded_body_size=self.read_uint(reader), - url=self.read_string(reader), - initiator=self.read_string(reader), - transferred_size=self.read_uint(reader), - cached=self.read_boolean(reader) + timestamp=MessageCodec.read_uint(reader), + duration=MessageCodec.read_uint(reader), + ttfb=MessageCodec.read_uint(reader), + header_size=MessageCodec.read_uint(reader), + encoded_body_size=MessageCodec.read_uint(reader), + decoded_body_size=MessageCodec.read_uint(reader), + url=MessageCodec.read_string(reader), + initiator=MessageCodec.read_string(reader), + transferred_size=MessageCodec.read_uint(reader), + cached=MessageCodec.read_boolean(reader) ) if message_id == 117: return TabChange( - tab_id=self.read_string(reader) + tab_id=MessageCodec.read_string(reader) ) if message_id == 118: return TabData( - tab_id=self.read_string(reader) + tab_id=MessageCodec.read_string(reader) ) if message_id == 125: return IssueEvent( - message_id=self.read_uint(reader), - timestamp=self.read_uint(reader), - type=self.read_string(reader), - context_string=self.read_string(reader), - context=self.read_string(reader), - payload=self.read_string(reader), - url=self.read_string(reader) + message_id=MessageCodec.read_uint(reader), + timestamp=MessageCodec.read_uint(reader), + type=MessageCodec.read_string(reader), + context_string=MessageCodec.read_string(reader), + context=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader), + url=MessageCodec.read_string(reader) ) if message_id == 126: return SessionEnd( - timestamp=self.read_uint(reader), - encryption_key=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + encryption_key=MessageCodec.read_string(reader) ) if message_id == 127: return SessionSearch( - timestamp=self.read_uint(reader), - partition=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader), + partition=MessageCodec.read_uint(reader) ) if message_id == 90: return IOSSessionStart( - timestamp=self.read_uint(reader), - project_id=self.read_uint(reader), - tracker_version=self.read_string(reader), - rev_id=self.read_string(reader), - user_uuid=self.read_string(reader), - user_os=self.read_string(reader), - user_os_version=self.read_string(reader), - user_device=self.read_string(reader), - user_device_type=self.read_string(reader), - user_country=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + project_id=MessageCodec.read_uint(reader), + tracker_version=MessageCodec.read_string(reader), + rev_id=MessageCodec.read_string(reader), + user_uuid=MessageCodec.read_string(reader), + user_os=MessageCodec.read_string(reader), + user_os_version=MessageCodec.read_string(reader), + user_device=MessageCodec.read_string(reader), + user_device_type=MessageCodec.read_string(reader), + user_country=MessageCodec.read_string(reader) ) if message_id == 91: return IOSSessionEnd( - timestamp=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader) ) if message_id == 92: return IOSMetadata( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - key=self.read_string(reader), - value=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + key=MessageCodec.read_string(reader), + value=MessageCodec.read_string(reader) ) if message_id == 93: return IOSCustomEvent( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - name=self.read_string(reader), - payload=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + name=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader) ) if message_id == 94: return IOSUserID( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - value=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + value=MessageCodec.read_string(reader) ) if message_id == 95: return IOSUserAnonymousID( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - value=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + value=MessageCodec.read_string(reader) ) if message_id == 96: return IOSScreenChanges( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - x=self.read_uint(reader), - y=self.read_uint(reader), - width=self.read_uint(reader), - height=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + x=MessageCodec.read_uint(reader), + y=MessageCodec.read_uint(reader), + width=MessageCodec.read_uint(reader), + height=MessageCodec.read_uint(reader) ) if message_id == 97: return IOSCrash( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - name=self.read_string(reader), - reason=self.read_string(reader), - stacktrace=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + name=MessageCodec.read_string(reader), + reason=MessageCodec.read_string(reader), + stacktrace=MessageCodec.read_string(reader) ) if message_id == 98: return IOSViewComponentEvent( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - screen_name=self.read_string(reader), - view_name=self.read_string(reader), - visible=self.read_boolean(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + screen_name=MessageCodec.read_string(reader), + view_name=MessageCodec.read_string(reader), + visible=MessageCodec.read_boolean(reader) ) if message_id == 100: return IOSClickEvent( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - label=self.read_string(reader), - x=self.read_uint(reader), - y=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + label=MessageCodec.read_string(reader), + x=MessageCodec.read_uint(reader), + y=MessageCodec.read_uint(reader) ) if message_id == 101: return IOSInputEvent( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - value=self.read_string(reader), - value_masked=self.read_boolean(reader), - label=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + value=MessageCodec.read_string(reader), + value_masked=MessageCodec.read_boolean(reader), + label=MessageCodec.read_string(reader) ) if message_id == 102: return IOSPerformanceEvent( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - name=self.read_string(reader), - value=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + name=MessageCodec.read_string(reader), + value=MessageCodec.read_uint(reader) ) if message_id == 103: return IOSLog( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - severity=self.read_string(reader), - content=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + severity=MessageCodec.read_string(reader), + content=MessageCodec.read_string(reader) ) if message_id == 104: return IOSInternalError( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - content=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + content=MessageCodec.read_string(reader) ) if message_id == 105: return IOSNetworkCall( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - type=self.read_string(reader), - method=self.read_string(reader), - url=self.read_string(reader), - request=self.read_string(reader), - response=self.read_string(reader), - status=self.read_uint(reader), - duration=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + type=MessageCodec.read_string(reader), + method=MessageCodec.read_string(reader), + url=MessageCodec.read_string(reader), + request=MessageCodec.read_string(reader), + response=MessageCodec.read_string(reader), + status=MessageCodec.read_uint(reader), + duration=MessageCodec.read_uint(reader) ) if message_id == 106: return IOSSwipeEvent( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - label=self.read_string(reader), - x=self.read_uint(reader), - y=self.read_uint(reader), - direction=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + label=MessageCodec.read_string(reader), + x=MessageCodec.read_uint(reader), + y=MessageCodec.read_uint(reader), + direction=MessageCodec.read_string(reader) ) if message_id == 107: return IOSBatchMeta( - timestamp=self.read_uint(reader), - length=self.read_uint(reader), - first_index=self.read_uint(reader) + timestamp=MessageCodec.read_uint(reader), + length=MessageCodec.read_uint(reader), + first_index=MessageCodec.read_uint(reader) ) if message_id == 110: return IOSPerformanceAggregated( - timestamp_start=self.read_uint(reader), - timestamp_end=self.read_uint(reader), - min_fps=self.read_uint(reader), - avg_fps=self.read_uint(reader), - max_fps=self.read_uint(reader), - min_cpu=self.read_uint(reader), - avg_cpu=self.read_uint(reader), - max_cpu=self.read_uint(reader), - min_memory=self.read_uint(reader), - avg_memory=self.read_uint(reader), - max_memory=self.read_uint(reader), - min_battery=self.read_uint(reader), - avg_battery=self.read_uint(reader), - max_battery=self.read_uint(reader) + timestamp_start=MessageCodec.read_uint(reader), + timestamp_end=MessageCodec.read_uint(reader), + min_fps=MessageCodec.read_uint(reader), + avg_fps=MessageCodec.read_uint(reader), + max_fps=MessageCodec.read_uint(reader), + min_cpu=MessageCodec.read_uint(reader), + avg_cpu=MessageCodec.read_uint(reader), + max_cpu=MessageCodec.read_uint(reader), + min_memory=MessageCodec.read_uint(reader), + avg_memory=MessageCodec.read_uint(reader), + max_memory=MessageCodec.read_uint(reader), + min_battery=MessageCodec.read_uint(reader), + avg_battery=MessageCodec.read_uint(reader), + max_battery=MessageCodec.read_uint(reader) ) if message_id == 111: return IOSIssueEvent( - timestamp=self.read_uint(reader), - type=self.read_string(reader), - context_string=self.read_string(reader), - context=self.read_string(reader), - payload=self.read_string(reader) + timestamp=MessageCodec.read_uint(reader), + type=MessageCodec.read_string(reader), + context_string=MessageCodec.read_string(reader), + context=MessageCodec.read_string(reader), + payload=MessageCodec.read_string(reader) ) diff --git a/ee/connectors/requirements-fill.txt b/ee/connectors/requirements-fill.txt deleted file mode 100644 index 1d69f0ca8..000000000 --- a/ee/connectors/requirements-fill.txt +++ /dev/null @@ -1,13 +0,0 @@ -chardet==5.1.0 -idna==3.4 -psycopg2-binary==2.9.6 -python-decouple==3.8 -pytz==2022.6 -requests==2.28.1 -SQLAlchemy==1.4.48 -tzlocal==5.0.1 -urllib3==1.26.15 -sqlalchemy-redshift==0.8.14 -redshift-connector==2.0.911 -pandas-redshift==2.0.5 -PyYAML==6.0 diff --git a/ee/connectors/sql/redshift_events.sql b/ee/connectors/sql/redshift_events.sql index 48213a50a..84191ca9a 100644 --- a/ee/connectors/sql/redshift_events.sql +++ b/ee/connectors/sql/redshift_events.sql @@ -3,6 +3,10 @@ CREATE TABLE IF NOT EXISTS connector_events sessionid BIGINT, consolelog_level VARCHAR(8000), consolelog_value VARCHAR(8000), + clickevent_hesitationtime BIGINT, + clickevent_messageid BIGINT, + clickevent_label VARCHAR(8000), + clickevent_selector VARCHAR(8000), customevent_name VARCHAR(8000), customevent_payload VARCHAR(8000), jsexception_message VARCHAR(8000), diff --git a/ee/connectors/utils/worker.py b/ee/connectors/utils/worker.py index 8832594d0..fab10a101 100644 --- a/ee/connectors/utils/worker.py +++ b/ee/connectors/utils/worker.py @@ -40,7 +40,7 @@ if ssl_protocol: session_messages = [1, 25, 28, 29, 30, 31, 32, 54, 56, 62, 69, 78, 125, 126] if EVENT_TYPE == 'normal': - events_messages = [21, 22, 25, 27, 64, 78, 125] + events_messages = [21, 22, 25, 27, 64, 69, 78, 125] elif EVENT_TYPE == 'detailed': events_messages = [1, 4, 21, 22, 25, 27, 31, 32, 39, 48, 59, 64, 69, 78, 125, 126] allowed_messages = list(set(session_messages + events_messages))