From 7b9cc0fca65bff717a10ac07f542c17435a68887 Mon Sep 17 00:00:00 2001 From: sylenien Date: Tue, 26 Jul 2022 17:05:55 +0200 Subject: [PATCH 01/43] fix(tracker): fix assist typings --- tracker/tracker-assist/src/Assist.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index a36c307db..44b90c848 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -14,7 +14,7 @@ import type { Options as ConfirmOptions } from './ConfirmWindow/defaults.js'; // TODO: fully specified strict check (everywhere) -type StartEndCallback = () => ((()=>{}) | void) +type StartEndCallback = () => ((() => void) | void) export interface Options { onAgentConnect: StartEndCallback, @@ -39,8 +39,8 @@ enum CallingState { }; -// TODO typing???? -type OptionalCallback = (()=>{}) | void +// TODO typing???? () => ((() => void) | void) +type OptionalCallback = (() => void) | void type Agent = { onDisconnect?: OptionalCallback, onControlReleased?: OptionalCallback, From 819aefb86efbf8f721f2a4a0cf86d01f289aa025 Mon Sep 17 00:00:00 2001 From: sylenien Date: Tue, 26 Jul 2022 17:06:48 +0200 Subject: [PATCH 02/43] fix(tracker): fix assist typings --- tracker/tracker-assist/src/Assist.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 44b90c848..43f342481 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -39,7 +39,7 @@ enum CallingState { }; -// TODO typing???? () => ((() => void) | void) +// TODO typing???? type OptionalCallback = (() => void) | void type Agent = { onDisconnect?: OptionalCallback, From 772261fef4ea9ea7b6f237b062dede3d54d670dc Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers <7943856+bschoenmaeckers@users.noreply.github.com> Date: Wed, 27 Jul 2022 18:40:45 +0200 Subject: [PATCH 03/43] fix(ui): Serve react index file for all page urls (#642) --- frontend/nginx.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 0138027f0..f5581606b 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -3,6 +3,9 @@ server { listen [::]:8080 default_server; root /var/www/openreplay; index index.html; + rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /index.html break; + proxy_intercept_errors on; # see frontend://nginx.org/en/docs/frontend/ngx_frontend_proxy_module.html#proxy_intercept_errors + error_page 404 =200 /index.html; location / { try_files $uri $uri/ =404; } From 976a7a5a036dbaf22596a5718b42d6fdc444f251 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 27 Jul 2022 18:48:58 +0200 Subject: [PATCH 04/43] feat(chalice): removed elasticsearch dependency --- api/chalicelib/core/log_tool_elasticsearch.py | 71 ++++++++++--------- api/requirements-alerts.txt | 1 - api/requirements.txt | 1 - ee/api/requirements-alerts.txt | 1 - ee/api/requirements-crons.txt | 1 - ee/api/requirements.txt | 1 - 6 files changed, 36 insertions(+), 40 deletions(-) diff --git a/api/chalicelib/core/log_tool_elasticsearch.py b/api/chalicelib/core/log_tool_elasticsearch.py index ba20636c4..90986f61d 100644 --- a/api/chalicelib/core/log_tool_elasticsearch.py +++ b/api/chalicelib/core/log_tool_elasticsearch.py @@ -1,7 +1,7 @@ # from elasticsearch import Elasticsearch, RequestsHttpConnection -from elasticsearch import Elasticsearch +# from elasticsearch import Elasticsearch from chalicelib.core import log_tools -import base64 +# import base64 import logging logging.getLogger('elasticsearch').level = logging.ERROR @@ -58,39 +58,40 @@ def add_edit(tenant_id, project_id, data): port=data["port"]) -def __get_es_client(host, port, api_key_id, api_key, use_ssl=False, timeout=15): - scheme = "http" if host.startswith("http") else "https" - host = host.replace("http://", "").replace("https://", "") - try: - args = { - "hosts": [{"host": host, "port": port, "scheme": scheme}], - "verify_certs": False, - # "ca_certs": False, - # "connection_class": RequestsHttpConnection, - "request_timeout": timeout, - "api_key": (api_key_id, api_key) - } - # if api_key_id is not None and len(api_key_id) > 0: - # # args["http_auth"] = (username, password) - # token = "ApiKey " + base64.b64encode(f"{api_key_id}:{api_key}".encode("utf-8")).decode("utf-8") - # args["headers"] = {"Authorization": token} - es = Elasticsearch( - **args - ) - r = es.ping() - if not r and not use_ssl: - return __get_es_client(host, port, api_key_id, api_key, use_ssl=True, timeout=timeout) - if not r: - return None - except Exception as err: - print("================exception connecting to ES host:") - print(err) - return None - return es +# def __get_es_client(host, port, api_key_id, api_key, use_ssl=False, timeout=15): +# scheme = "http" if host.startswith("http") else "https" +# host = host.replace("http://", "").replace("https://", "") +# try: +# args = { +# "hosts": [{"host": host, "port": port, "scheme": scheme}], +# "verify_certs": False, +# # "ca_certs": False, +# # "connection_class": RequestsHttpConnection, +# "request_timeout": timeout, +# "api_key": (api_key_id, api_key) +# } +# # if api_key_id is not None and len(api_key_id) > 0: +# # # args["http_auth"] = (username, password) +# # token = "ApiKey " + base64.b64encode(f"{api_key_id}:{api_key}".encode("utf-8")).decode("utf-8") +# # args["headers"] = {"Authorization": token} +# es = Elasticsearch( +# **args +# ) +# r = es.ping() +# if not r and not use_ssl: +# return __get_es_client(host, port, api_key_id, api_key, use_ssl=True, timeout=timeout) +# if not r: +# return None +# except Exception as err: +# print("================exception connecting to ES host:") +# print(err) +# return None +# return es def ping(tenant_id, host, port, apiKeyId, apiKey): - es = __get_es_client(host, port, apiKeyId, apiKey, timeout=3) - if es is None: - return {"state": False} - return {"state": es.ping()} + # es = __get_es_client(host, port, apiKeyId, apiKey, timeout=3) + # if es is None: + # return {"state": False} + # return {"state": es.ping()} + return {"state": True} diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index 81198b0f3..f70efdd13 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -3,7 +3,6 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 -elasticsearch==8.3.1 jira==3.3.0 diff --git a/api/requirements.txt b/api/requirements.txt index 81198b0f3..f70efdd13 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -3,7 +3,6 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 -elasticsearch==8.3.1 jira==3.3.0 diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index 66fa84713..07d31f369 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -3,7 +3,6 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 -elasticsearch==8.3.1 jira==3.3.0 diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 66fa84713..07d31f369 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -3,7 +3,6 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 -elasticsearch==8.3.1 jira==3.3.0 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 5ce044904..2a9ae2e1a 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -3,7 +3,6 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 -elasticsearch==8.3.1 jira==3.3.0 From 8cce90b8a7942f80998dd495cc37557bc8cdc8eb Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 28 Jul 2022 14:58:08 +0200 Subject: [PATCH 05/43] chore(helm): disable minio migration if s3 is enabled Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/templates/job.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index 8925b83d8..c0d7f0a45 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -103,6 +103,7 @@ spec: mountPath: /opt/openreplay - name: dbmigrationscript mountPath: /opt/migrations/ + {{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }} - name: minio image: bitnami/minio:2020.10.9-debian-10-r6 env: @@ -128,6 +129,7 @@ spec: mountPath: /opt/openreplay - name: dbmigrationscript mountPath: /opt/migrations/ + {{- end}} {{- if .Values.global.enterpriseEditionLicense }} # Enterprise migration - name: clickhouse From f1e4d60ea8fcec43bd0ba762e628a90d7108e0e7 Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 28 Jul 2022 16:34:44 +0200 Subject: [PATCH 06/43] DB improvements (#647) * feat(backend/db): updated ClickHouse library version from 1.5.4 to 2.2.0 * feat(backend/db): refactored ClickHouse connector * feat(backend/db): rewritten batch implementation for ClickHouse inserts * feat(backend/db): found and fix memory leak in db service --- backend/cmd/db/main.go | 11 +- backend/go.mod | 14 +- backend/go.sum | 52 +- ee/backend/internal/db/datasaver/stats.go | 2 +- ee/backend/pkg/db/clickhouse/bulk.go | 45 -- ee/backend/pkg/db/clickhouse/connector.go | 500 +++++++++++++++---- ee/backend/pkg/db/clickhouse/helpers.go | 34 -- ee/backend/pkg/db/clickhouse/messages-web.go | 243 --------- 8 files changed, 447 insertions(+), 454 deletions(-) delete mode 100644 ee/backend/pkg/db/clickhouse/bulk.go delete mode 100644 ee/backend/pkg/db/clickhouse/helpers.go delete mode 100644 ee/backend/pkg/db/clickhouse/messages-web.go diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 2e962cb1b..1712b8a3f 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -122,11 +122,18 @@ func main() { os.Exit(0) case <-commitTick: // Send collected batches to db + start := time.Now() pg.CommitBatches() + pgDur := time.Now().Sub(start).Milliseconds() + + start = time.Now() if err := saver.CommitStats(); err != nil { log.Printf("Error on stats commit: %v", err) } - // TODO?: separate stats & regular messages + chDur := time.Now().Sub(start).Milliseconds() + log.Printf("commit duration(ms), pg: %d, ch: %d", pgDur, chDur) + + // TODO: use commit worker to save time each tick if err := consumer.Commit(); err != nil { log.Printf("Error on consumer commit: %v", err) } @@ -134,7 +141,7 @@ func main() { // Handle new message from queue err := consumer.ConsumeNext() if err != nil { - log.Fatalf("Error on consumption: %v", err) // TODO: is always fatal? + log.Fatalf("Error on consumption: %v", err) } } } diff --git a/backend/go.mod b/backend/go.mod index a15e23196..caaf1bf83 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,12 +4,12 @@ go 1.18 require ( cloud.google.com/go/logging v1.4.2 - github.com/ClickHouse/clickhouse-go v1.5.4 + github.com/ClickHouse/clickhouse-go/v2 v2.2.0 github.com/aws/aws-sdk-go v1.35.23 github.com/btcsuite/btcutil v1.0.2 github.com/elastic/go-elasticsearch/v7 v7.13.1 github.com/go-redis/redis v6.15.9+incompatible - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/jackc/pgconn v1.6.0 github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 @@ -36,7 +36,6 @@ require ( cloud.google.com/go/storage v1.14.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/confluentinc/confluent-kafka-go v1.9.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -50,15 +49,19 @@ require ( github.com/jackc/pgproto3/v2 v2.0.2 // indirect github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 // indirect github.com/jackc/pgtype v1.3.0 // indirect - github.com/jackc/puddle v1.1.0 // indirect + github.com/jackc/puddle v1.2.2-0.20220404125616-4e959849469a // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.11.9 // indirect + github.com/klauspost/compress v1.15.7 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/paulmach/orb v0.7.1 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + github.com/stretchr/testify v1.8.0 // indirect go.opencensus.io v0.23.0 // indirect go.opentelemetry.io/otel/sdk v1.7.0 // indirect go.opentelemetry.io/otel/trace v1.7.0 // indirect @@ -73,5 +76,4 @@ require ( google.golang.org/grpc v1.46.2 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index 433f2b895..6b76d1278 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -61,9 +61,11 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/ClickHouse/clickhouse-go/v2 v2.2.0 h1:dj00TDKY+xwuTJdbpspCSmTLFyWzRJerTHwaBxut1C0= +github.com/ClickHouse/clickhouse-go/v2 v2.2.0/go.mod h1:8f2XZUi7XoeU+uPIytSi1cvx8fmJxi7vIgqpvYTF1+o= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -79,7 +81,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -100,7 +101,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -151,6 +151,8 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -158,6 +160,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -230,8 +233,9 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -240,8 +244,10 @@ github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/Oth github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -290,8 +296,9 @@ github.com/jackc/pgx/v4 v4.6.0 h1:Fh0O9GdlG4gYpjpwOqjdEodJUQM9jzN3Hdv7PN0xmm0= github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.0 h1:musOWczZC/rSbqut475Vfcczg7jJsdUQf0D6oKPLgNU= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.2-0.20220404125616-4e959849469a h1:oH7y/b+q2BEerCnARr/HZc1NxOYbKSJor4MqQXlhh+s= +github.com/jackc/puddle v1.2.2-0.20220404125616-4e959849469a/go.mod h1:ZQuO1Un86Xpe1ShKl08ERTzYhzWq+OvrvotbpeE3XO0= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -308,10 +315,11 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.11.9 h1:5OCMOdde1TCT2sookEuVeEZzA8bmRSFV3AwPDZAG8AA= -github.com/klauspost/compress v1.11.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok= +github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -339,6 +347,7 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -353,8 +362,12 @@ github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/oschwald/maxminddb-golang v1.7.0 h1:JmU4Q1WBv5Q+2KZy5xJI+98aUwTIrPPxZUkd5Cwr8Zc= github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/paulmach/orb v0.7.1 h1:Zha++Z5OX/l168sqHK3k4z18LDvr+YAO/VjK0ReQ9rU= +github.com/paulmach/orb v0.7.1/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -393,8 +406,12 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sethvargo/go-envconfig v0.7.0 h1:P/ljQXSRjgAgsnIripHs53Jg/uNVXu2FYQ9yLSDappA= github.com/sethvargo/go-envconfig v0.7.0/go.mod h1:00S1FAhRUuTNJazWBWcJGvEHOM+NO6DhoRMAOX7FY5o= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= +github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -403,14 +420,19 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= github.com/ua-parser/uap-go v0.0.0-20200325213135-e1c09f13e2fe h1:aj/vX5epIlQQBEocKoM9nSAiNpakdQzElc8SaRFPu+I= @@ -420,6 +442,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -594,8 +617,10 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -649,6 +674,7 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -706,6 +732,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -714,6 +741,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -927,8 +955,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/ee/backend/internal/db/datasaver/stats.go b/ee/backend/internal/db/datasaver/stats.go index d5bd74f83..7fa2fb9d0 100644 --- a/ee/backend/internal/db/datasaver/stats.go +++ b/ee/backend/internal/db/datasaver/stats.go @@ -10,7 +10,7 @@ import ( . "openreplay/backend/pkg/messages" ) -var ch *clickhouse.Connector +var ch clickhouse.Connector var finalizeTicker <-chan time.Time func (si *Saver) InitStats() { diff --git a/ee/backend/pkg/db/clickhouse/bulk.go b/ee/backend/pkg/db/clickhouse/bulk.go deleted file mode 100644 index 121cdbbf0..000000000 --- a/ee/backend/pkg/db/clickhouse/bulk.go +++ /dev/null @@ -1,45 +0,0 @@ -package clickhouse - -import ( - "errors" - "database/sql" -) - -type bulk struct { - db *sql.DB - query string - tx *sql.Tx - stmt *sql.Stmt -} - -func newBulk(db *sql.DB, query string) *bulk { - return &bulk{ - db: db, - query: query, - } -} - -func (b *bulk) prepare() error { - var err error - b.tx, err = b.db.Begin() - if err != nil { - return err - } - b.stmt, err = b.tx.Prepare(b.query) - if err != nil { - return err - } - return nil -} - -func (b *bulk) commit() error { - return b.tx.Commit() -} - -func (b *bulk) exec(args ...interface{}) error { - if b.stmt == nil { - return errors.New("Bulk is not prepared.") - } - _, err := b.stmt.Exec(args...) - return err -} diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index cc0d20497..1fd6e5d1e 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -1,138 +1,416 @@ package clickhouse import ( - "database/sql" - _ "github.com/ClickHouse/clickhouse-go" + "context" + "errors" + "fmt" + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "log" + "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/hashid" + "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/url" + "strings" + "time" "openreplay/backend/pkg/license" ) -type Connector struct { - sessions *bulk - metadata *bulk // TODO: join sessions, sessions_metadata & sessions_ios - resources *bulk - pages *bulk - clicks *bulk - inputs *bulk - errors *bulk - performance *bulk - longtasks *bulk - db *sql.DB +var CONTEXT_MAP = map[uint64]string{0: "unknown", 1: "self", 2: "same-origin-ancestor", 3: "same-origin-descendant", 4: "same-origin", 5: "cross-origin-ancestor", 6: "cross-origin-descendant", 7: "cross-origin-unreachable", 8: "multiple-contexts"} +var CONTAINER_TYPE_MAP = map[uint64]string{0: "window", 1: "iframe", 2: "embed", 3: "object"} + +type Connector interface { + Prepare() error + Commit() error + FinaliseSessionsTable() error + InsertWebSession(session *types.Session) error + InsertWebResourceEvent(session *types.Session, msg *messages.ResourceEvent) error + InsertWebPageEvent(session *types.Session, msg *messages.PageEvent) error + InsertWebClickEvent(session *types.Session, msg *messages.ClickEvent) error + InsertWebInputEvent(session *types.Session, msg *messages.InputEvent) error + InsertWebErrorEvent(session *types.Session, msg *messages.ErrorEvent) error + InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error + InsertLongtask(session *types.Session, msg *messages.LongTask) error } -func NewConnector(url string) *Connector { +type connectorImpl struct { + conn driver.Conn + batches map[string]driver.Batch +} + +func NewConnector(url string) Connector { license.CheckLicense() - - db, err := sql.Open("clickhouse", url) + url = strings.TrimPrefix(url, "tcp://") + url = strings.TrimSuffix(url, "/default") + conn, err := clickhouse.Open(&clickhouse.Options{ + Addr: []string{url}, + Auth: clickhouse.Auth{ + Database: "default", + }, + MaxOpenConns: 20, + MaxIdleConns: 15, + ConnMaxLifetime: 3 * time.Minute, + Compression: &clickhouse.Compression{ + Method: clickhouse.CompressionLZ4, + }, + // Debug: true, + }) if err != nil { - log.Fatalln(err) + log.Fatal(err) } - return &Connector{ - db: db, - sessions: newBulk(db, ` - INSERT INTO sessions (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, datetime, duration, pages_count, events_count, errors_count, user_browser, user_browser_version) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - // TODO: join sessions, sessions_metadata & sessions_ios - metadata: newBulk(db, ` - INSERT INTO sessions_metadata (session_id, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, datetime) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - resources: newBulk(db, ` - INSERT INTO resources (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - pages: newBulk(db, ` - INSERT INTO pages (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - clicks: newBulk(db, ` - INSERT INTO clicks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label, hesitation_time) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - inputs: newBulk(db, ` - INSERT INTO inputs (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - errors: newBulk(db, ` - INSERT INTO errors (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, source, name, message, error_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - performance: newBulk(db, ` - INSERT INTO performance (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - longtasks: newBulk(db, ` - INSERT INTO longtasks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, context, container_type, container_id, container_name, container_src) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), + + c := &connectorImpl{ + conn: conn, + batches: make(map[string]driver.Batch, 9), } + return c } -func (conn *Connector) Prepare() error { - if err := conn.sessions.prepare(); err != nil { - return err +func (c *connectorImpl) newBatch(name, query string) error { + batch, err := c.conn.PrepareBatch(context.Background(), query) + if err != nil { + return fmt.Errorf("can't create new batch: %s", err) } - if err := conn.metadata.prepare(); err != nil { - return err + if _, ok := c.batches[name]; ok { + delete(c.batches, name) } - if err := conn.resources.prepare(); err != nil { - return err - } - if err := conn.pages.prepare(); err != nil { - return err - } - if err := conn.clicks.prepare(); err != nil { - return err - } - if err := conn.inputs.prepare(); err != nil { - return err - } - if err := conn.errors.prepare(); err != nil { - return err - } - if err := conn.performance.prepare(); err != nil { - return err - } - if err := conn.longtasks.prepare(); err != nil { - return err + c.batches[name] = batch + return nil +} + +var batches = map[string]string{ + "sessions": "INSERT INTO sessions (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, datetime, duration, pages_count, events_count, errors_count, user_browser, user_browser_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "metadata": "INSERT INTO sessions_metadata (session_id, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, datetime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "resources": "INSERT INTO resources (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "pages": "INSERT INTO pages (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "clicks": "INSERT INTO clicks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label, hesitation_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "inputs": "INSERT INTO inputs (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "errors": "INSERT INTO errors (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, source, name, message, error_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "performance": "INSERT INTO performance (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "longtasks": "INSERT INTO longtasks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, context, container_type, container_id, container_name, container_src) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", +} + +func (c *connectorImpl) Prepare() error { + for table, query := range batches { + if err := c.newBatch(table, query); err != nil { + return fmt.Errorf("can't create %s batch: %s", table, err) + } } return nil } -func (conn *Connector) Commit() error { - if err := conn.sessions.commit(); err != nil { - return err - } - if err := conn.metadata.commit(); err != nil { - return err - } - if err := conn.resources.commit(); err != nil { - return err - } - if err := conn.pages.commit(); err != nil { - return err - } - if err := conn.clicks.commit(); err != nil { - return err - } - if err := conn.inputs.commit(); err != nil { - return err - } - if err := conn.errors.commit(); err != nil { - return err - } - if err := conn.performance.commit(); err != nil { - return err - } - if err := conn.longtasks.commit(); err != nil { - return err +func (c *connectorImpl) Commit() error { + for _, b := range c.batches { + if err := b.Send(); err != nil { + return fmt.Errorf("can't send batch: %s", err) + } } return nil } -func (conn *Connector) FinaliseSessionsTable() error { - _, err := conn.db.Exec("OPTIMIZE TABLE sessions FINAL") - return err +func (c *connectorImpl) FinaliseSessionsTable() error { + if err := c.conn.Exec(context.Background(), "OPTIMIZE TABLE sessions FINAL"); err != nil { + return fmt.Errorf("can't finalise sessions table: %s", err) + } + return nil +} + +func (c *connectorImpl) checkError(name string, err error) { + if err != clickhouse.ErrBatchAlreadySent { + if batchErr := c.newBatch(name, batches[name]); batchErr != nil { + log.Printf("can't create %s batch after failed append operation: %s", name, batchErr) + } + } +} + +func (c *connectorImpl) InsertWebSession(session *types.Session) error { + if session.Duration == nil { + return errors.New("trying to insert session with nil duration") + } + if err := c.batches["sessions"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(session.Timestamp), + uint32(*session.Duration), + uint16(session.PagesCount), + uint16(session.EventsCount), + uint16(session.ErrorsCount), + // Web unique columns + session.UserBrowser, + nullableString(session.UserBrowserVersion), + ); err != nil { + c.checkError("sessions", err) + return fmt.Errorf("can't append to sessions batch: %s", err) + } + if err := c.batches["metadata"].Append( + session.SessionID, + session.UserID, + session.UserAnonymousID, + session.Metadata1, + session.Metadata2, + session.Metadata3, + session.Metadata4, + session.Metadata5, + session.Metadata6, + session.Metadata7, + session.Metadata8, + session.Metadata9, + session.Metadata10, + datetime(session.Timestamp), + ); err != nil { + c.checkError("metadata", err) + return fmt.Errorf("can't append to metadata batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebResourceEvent(session *types.Session, msg *messages.ResourceEvent) error { + var method interface{} = url.EnsureMethod(msg.Method) + if method == "" { + method = nil + } + if err := c.batches["resources"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + url.DiscardURLQuery(msg.URL), + msg.Type, + nullableUint16(uint16(msg.Duration)), + nullableUint16(uint16(msg.TTFB)), + nullableUint16(uint16(msg.HeaderSize)), + nullableUint32(uint32(msg.EncodedBodySize)), + nullableUint32(uint32(msg.DecodedBodySize)), + msg.Success, + ); err != nil { + c.checkError("resources", err) + return fmt.Errorf("can't append to resources batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebPageEvent(session *types.Session, msg *messages.PageEvent) error { + if err := c.batches["pages"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + url.DiscardURLQuery(msg.URL), + nullableUint16(uint16(msg.RequestStart)), + nullableUint16(uint16(msg.ResponseStart)), + nullableUint16(uint16(msg.ResponseEnd)), + nullableUint16(uint16(msg.DomContentLoadedEventStart)), + nullableUint16(uint16(msg.DomContentLoadedEventEnd)), + nullableUint16(uint16(msg.LoadEventStart)), + nullableUint16(uint16(msg.LoadEventEnd)), + nullableUint16(uint16(msg.FirstPaint)), + nullableUint16(uint16(msg.FirstContentfulPaint)), + nullableUint16(uint16(msg.SpeedIndex)), + nullableUint16(uint16(msg.VisuallyComplete)), + nullableUint16(uint16(msg.TimeToInteractive)), + ); err != nil { + c.checkError("pages", err) + return fmt.Errorf("can't append to pages batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebClickEvent(session *types.Session, msg *messages.ClickEvent) error { + if msg.Label == "" { + return nil + } + if err := c.batches["clicks"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + msg.Label, + nullableUint32(uint32(msg.HesitationTime)), + ); err != nil { + c.checkError("clicks", err) + return fmt.Errorf("can't append to clicks batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebInputEvent(session *types.Session, msg *messages.InputEvent) error { + if msg.Label == "" { + return nil + } + if err := c.batches["inputs"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + msg.Label, + ); err != nil { + c.checkError("inputs", err) + return fmt.Errorf("can't append to inputs batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebErrorEvent(session *types.Session, msg *messages.ErrorEvent) error { + if err := c.batches["errors"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + msg.Source, + nullableString(msg.Name), + msg.Message, + hashid.WebErrorID(session.ProjectID, msg), + ); err != nil { + c.checkError("errors", err) + return fmt.Errorf("can't append to errors batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error { + var timestamp uint64 = (msg.TimestampStart + msg.TimestampEnd) / 2 + if err := c.batches["performance"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(timestamp), + uint8(msg.MinFPS), + uint8(msg.AvgFPS), + uint8(msg.MaxFPS), + uint8(msg.MinCPU), + uint8(msg.AvgCPU), + uint8(msg.MaxCPU), + msg.MinTotalJSHeapSize, + msg.AvgTotalJSHeapSize, + msg.MaxTotalJSHeapSize, + msg.MinUsedJSHeapSize, + msg.AvgUsedJSHeapSize, + msg.MaxUsedJSHeapSize, + ); err != nil { + c.checkError("performance", err) + return fmt.Errorf("can't append to performance batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertLongtask(session *types.Session, msg *messages.LongTask) error { + if err := c.batches["longtasks"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + CONTEXT_MAP[msg.Context], + CONTAINER_TYPE_MAP[msg.ContainerType], + msg.ContainerId, + msg.ContainerName, + msg.ContainerSrc, + ); err != nil { + c.checkError("longtasks", err) + return fmt.Errorf("can't append to longtasks batch: %s", err) + } + return nil +} + +func nullableUint16(v uint16) *uint16 { + var p *uint16 = nil + if v != 0 { + p = &v + } + return p +} + +func nullableUint32(v uint32) *uint32 { + var p *uint32 = nil + if v != 0 { + p = &v + } + return p +} + +func nullableString(v string) *string { + var p *string = nil + if v != "" { + p = &v + } + return p +} + +func datetime(timestamp uint64) time.Time { + t := time.Unix(int64(timestamp/1e3), 0) + // Temporal solution for not correct timestamps in performance messages + if t.Year() < 2022 || t.Year() > 2025 { + return time.Now() + } + return t } diff --git a/ee/backend/pkg/db/clickhouse/helpers.go b/ee/backend/pkg/db/clickhouse/helpers.go deleted file mode 100644 index 37e30518c..000000000 --- a/ee/backend/pkg/db/clickhouse/helpers.go +++ /dev/null @@ -1,34 +0,0 @@ -package clickhouse - -import ( - "time" -) - - -func nullableUint16(v uint16) *uint16 { - var p *uint16 = nil - if v != 0 { - p = &v - } - return p -} - -func nullableUint32(v uint32) *uint32 { - var p *uint32 = nil - if v != 0 { - p = &v - } - return p -} - -func nullableString(v string) *string { - var p *string = nil - if v != "" { - p = &v - } - return p -} - -func datetime(timestamp uint64) time.Time { - return time.Unix(int64(timestamp/1e3), 0) -} diff --git a/ee/backend/pkg/db/clickhouse/messages-web.go b/ee/backend/pkg/db/clickhouse/messages-web.go deleted file mode 100644 index adfa38655..000000000 --- a/ee/backend/pkg/db/clickhouse/messages-web.go +++ /dev/null @@ -1,243 +0,0 @@ -package clickhouse - -import ( - "errors" - - . "openreplay/backend/pkg/db/types" - "openreplay/backend/pkg/hashid" - . "openreplay/backend/pkg/messages" - "openreplay/backend/pkg/url" -) - -func (conn *Connector) InsertWebSession(session *Session) error { - if session.Duration == nil { - return errors.New("Clickhouse: trying to insert session with ") - } - - if err := conn.sessions.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(session.Timestamp), - uint32(*session.Duration), - session.PagesCount, - session.EventsCount, - session.ErrorsCount, - // Web unique columns - session.UserBrowser, - nullableString(session.UserBrowserVersion), - ); err != nil { - return err - } - // TODO: join sessions, sessions_metadata & sessions_ios - return conn.metadata.exec( - session.SessionID, - session.UserID, - session.UserAnonymousID, - session.Metadata1, - session.Metadata2, - session.Metadata3, - session.Metadata4, - session.Metadata5, - session.Metadata6, - session.Metadata7, - session.Metadata8, - session.Metadata9, - session.Metadata10, - datetime(session.Timestamp), - ) -} - -func (conn *Connector) InsertWebResourceEvent(session *Session, msg *ResourceEvent) error { - // nullableString causes error "unexpected type *string" on Nullable Enum type - // (apparently, a clickhouse-go bug) https://github.com/ClickHouse/clickhouse-go/pull/204 - var method interface{} = url.EnsureMethod(msg.Method) - if method == "" { - method = nil - } - return conn.resources.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - url.DiscardURLQuery(msg.URL), - msg.Type, - nullableUint16(uint16(msg.Duration)), - nullableUint16(uint16(msg.TTFB)), - nullableUint16(uint16(msg.HeaderSize)), - nullableUint32(uint32(msg.EncodedBodySize)), - nullableUint32(uint32(msg.DecodedBodySize)), - msg.Success, - ) -} - -func (conn *Connector) InsertWebPageEvent(session *Session, msg *PageEvent) error { - return conn.pages.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - url.DiscardURLQuery(msg.URL), - nullableUint16(uint16(msg.RequestStart)), - nullableUint16(uint16(msg.ResponseStart)), - nullableUint16(uint16(msg.ResponseEnd)), - nullableUint16(uint16(msg.DomContentLoadedEventStart)), - nullableUint16(uint16(msg.DomContentLoadedEventEnd)), - nullableUint16(uint16(msg.LoadEventStart)), - nullableUint16(uint16(msg.LoadEventEnd)), - nullableUint16(uint16(msg.FirstPaint)), - nullableUint16(uint16(msg.FirstContentfulPaint)), - nullableUint16(uint16(msg.SpeedIndex)), - nullableUint16(uint16(msg.VisuallyComplete)), - nullableUint16(uint16(msg.TimeToInteractive)), - ) -} - -func (conn *Connector) InsertWebClickEvent(session *Session, msg *ClickEvent) error { - if msg.Label == "" { - return nil - } - return conn.clicks.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - msg.Label, - nullableUint32(uint32(msg.HesitationTime)), - ) -} - -func (conn *Connector) InsertWebInputEvent(session *Session, msg *InputEvent) error { - if msg.Label == "" { - return nil - } - return conn.inputs.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - msg.Label, - ) -} - -func (conn *Connector) InsertWebErrorEvent(session *Session, msg *ErrorEvent) error { - return conn.errors.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - msg.Source, - nullableString(msg.Name), - msg.Message, - hashid.WebErrorID(session.ProjectID, msg), - ) -} - -func (conn *Connector) InsertWebPerformanceTrackAggr(session *Session, msg *PerformanceTrackAggr) error { - var timestamp uint64 = (msg.TimestampStart + msg.TimestampEnd) / 2 - return conn.performance.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(timestamp), - uint8(msg.MinFPS), - uint8(msg.AvgFPS), - uint8(msg.MaxFPS), - uint8(msg.MinCPU), - uint8(msg.AvgCPU), - uint8(msg.MaxCPU), - msg.MinTotalJSHeapSize, - msg.AvgTotalJSHeapSize, - msg.MaxTotalJSHeapSize, - msg.MinUsedJSHeapSize, - msg.AvgUsedJSHeapSize, - msg.MaxUsedJSHeapSize, - ) -} - -// TODO: make enum message type -var CONTEXT_MAP = map[uint64]string{0: "unknown", 1: "self", 2: "same-origin-ancestor", 3: "same-origin-descendant", 4: "same-origin", 5: "cross-origin-ancestor", 6: "cross-origin-descendant", 7: "cross-origin-unreachable", 8: "multiple-contexts"} -var CONTAINER_TYPE_MAP = map[uint64]string{0: "window", 1: "iframe", 2: "embed", 3: "object"} - -func (conn *Connector) InsertLongtask(session *Session, msg *LongTask) error { - return conn.longtasks.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - CONTEXT_MAP[msg.Context], - CONTAINER_TYPE_MAP[msg.ContainerType], - msg.ContainerId, - msg.ContainerName, - msg.ContainerSrc, - ) -} From 959d54be7ef1a2dcf3f2f3441910e189b8ec5899 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 1 Aug 2022 18:05:38 +0200 Subject: [PATCH 07/43] feat(chalice): fixed update self-user feat(chalice): optimized /projects --- api/chalicelib/core/projects.py | 28 ++++++++++++++++++---------- api/routers/core.py | 8 -------- api/routers/core_dynamic.py | 8 ++++++++ ee/api/chalicelib/core/projects.py | 24 +++++++++++------------- ee/api/routers/core_dynamic.py | 8 ++++++++ 5 files changed, 45 insertions(+), 31 deletions(-) diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index 0b0bd963f..0893f6259 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -43,16 +43,24 @@ def __create(tenant_id, name): def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, stack_integrations=False): with pg_client.PostgresClient() as cur: - cur.execute(f"""\ - SELECT - s.project_id, s.name, s.project_key, s.save_request_payloads - {',s.gdpr' if gdpr else ''} - {',COALESCE((SELECT TRUE FROM public.sessions WHERE sessions.project_id = s.project_id LIMIT 1), FALSE) AS recorded' if recorded else ''} - {',stack_integrations.count>0 AS stack_integrations' if stack_integrations else ''} - FROM public.projects AS s - {'LEFT JOIN LATERAL (SELECT COUNT(*) AS count FROM public.integrations WHERE s.project_id = integrations.project_id LIMIT 1) AS stack_integrations ON TRUE' if stack_integrations else ''} - WHERE s.deleted_at IS NULL - ORDER BY s.project_id;""") + recorded_q = "" + if recorded: + recorded_q = """, COALESCE((SELECT TRUE + FROM public.sessions + WHERE sessions.project_id = s.project_id + AND sessions.start_ts >= (EXTRACT(EPOCH FROM s.created_at) * 1000 - 24 * 60 * 60 * 1000) + AND sessions.start_ts <= %(now)s + LIMIT 1), FALSE) AS recorded""" + query = cur.mogrify(f"""SELECT + s.project_id, s.name, s.project_key, s.save_request_payloads + {',s.gdpr' if gdpr else ''} + {recorded_q} + {',stack_integrations.count>0 AS stack_integrations' if stack_integrations else ''} + FROM public.projects AS s + {'LEFT JOIN LATERAL (SELECT COUNT(*) AS count FROM public.integrations WHERE s.project_id = integrations.project_id LIMIT 1) AS stack_integrations ON TRUE' if stack_integrations else ''} + WHERE s.deleted_at IS NULL + ORDER BY s.project_id;""", {"now": TimeUTC.now()}) + cur.execute(query) rows = cur.fetchall() if recording_state: project_ids = [f'({r["project_id"]})' for r in rows] diff --git a/api/routers/core.py b/api/routers/core.py index 2c3ff5b90..c69c7ff43 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1110,14 +1110,6 @@ def generate_new_user_token(context: schemas.CurrentContext = Depends(OR_context return {"data": users.generate_new_api_key(user_id=context.user_id)} -@app.post('/account', tags=["account"]) -@app.put('/account', tags=["account"]) -def edit_account(data: schemas.EditUserSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return users.edit(tenant_id=context.tenant_id, user_id_to_update=context.user_id, changes=data, - editor_id=context.user_id) - - @app.post('/account/password', tags=["account"]) @app.put('/account/password', tags=["account"]) def change_client_password(data: schemas.EditUserPasswordSchema = Body(...), diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index 594715bb6..32eb78e41 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -43,6 +43,14 @@ def get_account(context: schemas.CurrentContext = Depends(OR_context)): } +@app.post('/account', tags=["account"]) +@app.put('/account', tags=["account"]) +def edit_account(data: schemas.EditUserSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return users.edit(tenant_id=context.tenant_id, user_id_to_update=context.user_id, changes=data, + editor_id=context.user_id) + + @app.get('/projects/limit', tags=['projects']) def get_projects_limit(context: schemas.CurrentContext = Depends(OR_context)): return {"data": { diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index e6ef34760..6700173b5 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -52,30 +52,28 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st AND users.tenant_id = %(tenant_id)s AND (roles.all_projects OR roles_projects.project_id = s.project_id) ) AS role_project ON (TRUE)""" - pre_select = "" + recorded_q = "" if recorded: - pre_select = """WITH recorded_p AS (SELECT DISTINCT projects.project_id - FROM projects INNER JOIN sessions USING (project_id) - WHERE tenant_id =%(tenant_id)s - AND deleted_at IS NULL - AND duration > 0)""" - cur.execute( - cur.mogrify(f"""\ - {pre_select} + recorded_q = """, COALESCE((SELECT TRUE + FROM public.sessions + WHERE sessions.project_id = s.project_id + AND sessions.start_ts >= (EXTRACT(EPOCH FROM s.created_at) * 1000 - 24 * 60 * 60 * 1000) + AND sessions.start_ts <= %(now)s + LIMIT 1), FALSE) AS recorded""" + query = cur.mogrify(f"""\ SELECT s.project_id, s.name, s.project_key, s.save_request_payloads {',s.gdpr' if gdpr else ''} - {',EXISTS(SELECT 1 FROM recorded_p WHERE recorded_p.project_id = s.project_id) AS recorded' if recorded else ''} + {recorded_q} {',stack_integrations.count>0 AS stack_integrations' if stack_integrations else ''} FROM public.projects AS s - {'LEFT JOIN recorded_p USING (project_id)' if recorded else ''} {'LEFT JOIN LATERAL (SELECT COUNT(*) AS count FROM public.integrations WHERE s.project_id = integrations.project_id LIMIT 1) AS stack_integrations ON TRUE' if stack_integrations else ''} {role_query if user_id is not None else ""} WHERE s.tenant_id =%(tenant_id)s AND s.deleted_at IS NULL ORDER BY s.project_id;""", - {"tenant_id": tenant_id, "user_id": user_id}) - ) + {"tenant_id": tenant_id, "user_id": user_id, "now": TimeUTC.now()}) + cur.execute(query) rows = cur.fetchall() if recording_state: project_ids = [f'({r["project_id"]})' for r in rows] diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index 3c5c21905..6d8470444 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -46,6 +46,14 @@ def get_account(context: schemas.CurrentContext = Depends(OR_context)): } +@app.post('/account', tags=["account"]) +@app.put('/account', tags=["account"]) +def edit_account(data: schemas_ee.EditUserSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return users.edit(tenant_id=context.tenant_id, user_id_to_update=context.user_id, changes=data, + editor_id=context.user_id) + + @app.get('/projects/limit', tags=['projects']) def get_projects_limit(context: schemas.CurrentContext = Depends(OR_context)): return {"data": { From cc73278f9ad0dc933f63c85549cbf3f5c3207527 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Mon, 1 Aug 2022 18:23:42 +0200 Subject: [PATCH 08/43] Revert "v1.7.0 ES change" --- api/chalicelib/core/log_tool_elasticsearch.py | 71 +++++++++---------- api/requirements-alerts.txt | 1 + api/requirements.txt | 1 + ee/api/requirements-alerts.txt | 1 + ee/api/requirements-crons.txt | 1 + ee/api/requirements.txt | 1 + 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/api/chalicelib/core/log_tool_elasticsearch.py b/api/chalicelib/core/log_tool_elasticsearch.py index 90986f61d..ba20636c4 100644 --- a/api/chalicelib/core/log_tool_elasticsearch.py +++ b/api/chalicelib/core/log_tool_elasticsearch.py @@ -1,7 +1,7 @@ # from elasticsearch import Elasticsearch, RequestsHttpConnection -# from elasticsearch import Elasticsearch +from elasticsearch import Elasticsearch from chalicelib.core import log_tools -# import base64 +import base64 import logging logging.getLogger('elasticsearch').level = logging.ERROR @@ -58,40 +58,39 @@ def add_edit(tenant_id, project_id, data): port=data["port"]) -# def __get_es_client(host, port, api_key_id, api_key, use_ssl=False, timeout=15): -# scheme = "http" if host.startswith("http") else "https" -# host = host.replace("http://", "").replace("https://", "") -# try: -# args = { -# "hosts": [{"host": host, "port": port, "scheme": scheme}], -# "verify_certs": False, -# # "ca_certs": False, -# # "connection_class": RequestsHttpConnection, -# "request_timeout": timeout, -# "api_key": (api_key_id, api_key) -# } -# # if api_key_id is not None and len(api_key_id) > 0: -# # # args["http_auth"] = (username, password) -# # token = "ApiKey " + base64.b64encode(f"{api_key_id}:{api_key}".encode("utf-8")).decode("utf-8") -# # args["headers"] = {"Authorization": token} -# es = Elasticsearch( -# **args -# ) -# r = es.ping() -# if not r and not use_ssl: -# return __get_es_client(host, port, api_key_id, api_key, use_ssl=True, timeout=timeout) -# if not r: -# return None -# except Exception as err: -# print("================exception connecting to ES host:") -# print(err) -# return None -# return es +def __get_es_client(host, port, api_key_id, api_key, use_ssl=False, timeout=15): + scheme = "http" if host.startswith("http") else "https" + host = host.replace("http://", "").replace("https://", "") + try: + args = { + "hosts": [{"host": host, "port": port, "scheme": scheme}], + "verify_certs": False, + # "ca_certs": False, + # "connection_class": RequestsHttpConnection, + "request_timeout": timeout, + "api_key": (api_key_id, api_key) + } + # if api_key_id is not None and len(api_key_id) > 0: + # # args["http_auth"] = (username, password) + # token = "ApiKey " + base64.b64encode(f"{api_key_id}:{api_key}".encode("utf-8")).decode("utf-8") + # args["headers"] = {"Authorization": token} + es = Elasticsearch( + **args + ) + r = es.ping() + if not r and not use_ssl: + return __get_es_client(host, port, api_key_id, api_key, use_ssl=True, timeout=timeout) + if not r: + return None + except Exception as err: + print("================exception connecting to ES host:") + print(err) + return None + return es def ping(tenant_id, host, port, apiKeyId, apiKey): - # es = __get_es_client(host, port, apiKeyId, apiKey, timeout=3) - # if es is None: - # return {"state": False} - # return {"state": es.ping()} - return {"state": True} + es = __get_es_client(host, port, apiKeyId, apiKey, timeout=3) + if es is None: + return {"state": False} + return {"state": es.ping()} diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index f70efdd13..81198b0f3 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -3,6 +3,7 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 +elasticsearch==8.3.1 jira==3.3.0 diff --git a/api/requirements.txt b/api/requirements.txt index f70efdd13..81198b0f3 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -3,6 +3,7 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 +elasticsearch==8.3.1 jira==3.3.0 diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index 07d31f369..66fa84713 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -3,6 +3,7 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 +elasticsearch==8.3.1 jira==3.3.0 diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 07d31f369..66fa84713 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -3,6 +3,7 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 +elasticsearch==8.3.1 jira==3.3.0 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 2a9ae2e1a..5ce044904 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -3,6 +3,7 @@ urllib3==1.26.10 boto3==1.24.26 pyjwt==2.4.0 psycopg2-binary==2.9.3 +elasticsearch==8.3.1 jira==3.3.0 From dddda50e56b923a8c824a4f899909a65b7181ea3 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 7 Jul 2022 17:15:08 +0200 Subject: [PATCH 09/43] chore(helm): Adding PV support for deployments --- .../charts/alerts/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/alerts/values.yaml | 12 ++++++++++++ .../charts/assets/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/assets/values.yaml | 12 ++++++++++++ .../charts/assist/templates/deployment.yaml | 10 +++++++++- .../helmcharts/openreplay/charts/assist/values.yaml | 12 ++++++++++++ .../charts/chalice/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/chalice/values.yaml | 12 ++++++++++++ .../openreplay/charts/db/templates/deployment.yaml | 8 ++++++++ scripts/helmcharts/openreplay/charts/db/values.yaml | 12 ++++++++++++ .../charts/ender/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/ender/values.yaml | 12 ++++++++++++ .../openreplay/charts/http/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/http/values.yaml | 12 ++++++++++++ .../charts/integrations/templates/deployment.yaml | 8 ++++++++ .../openreplay/charts/integrations/values.yaml | 12 ++++++++++++ .../charts/peers/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/peers/values.yaml | 12 ++++++++++++ 18 files changed, 181 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml index eac304df0..afb7aedc5 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml @@ -96,8 +96,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/alerts/values.yaml b/scripts/helmcharts/openreplay/charts/alerts/values.yaml index 8efd36a80..4bcc516c8 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/values.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/values.yaml @@ -99,3 +99,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml index ce0c41c99..5fbd084c0 100644 --- a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml @@ -90,8 +90,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/assets/values.yaml b/scripts/helmcharts/openreplay/charts/assets/values.yaml index e590f1b3c..2597ed36c 100644 --- a/scripts/helmcharts/openreplay/charts/assets/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/values.yaml @@ -96,3 +96,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml index 2a776ab58..ed4ec5d4a 100644 --- a/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml @@ -62,10 +62,18 @@ spec: {{- range $key, $val := .Values.service.ports }} - name: {{ $key }} containerPort: {{ $val }} - protocol: TCP {{- end }} + protocol: TCP + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/assist/values.yaml b/scripts/helmcharts/openreplay/charts/assist/values.yaml index 4ffd45a0d..4ffaf88e1 100644 --- a/scripts/helmcharts/openreplay/charts/assist/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/values.yaml @@ -97,3 +97,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index d2aafc35a..4491a82e4 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -106,8 +106,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 99acdbf76..2c9d75040 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -120,3 +120,15 @@ healthCheck: initialDelaySeconds: 100 periodSeconds: 15 timeoutSeconds: 10 + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml index 7afbf0e7d..2c18179df 100644 --- a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml @@ -62,8 +62,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/db/values.yaml b/scripts/helmcharts/openreplay/charts/db/values.yaml index 0da7ab913..7d375c594 100644 --- a/scripts/helmcharts/openreplay/charts/db/values.yaml +++ b/scripts/helmcharts/openreplay/charts/db/values.yaml @@ -98,3 +98,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml index 368c3ee29..a313415c8 100644 --- a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml @@ -62,8 +62,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/ender/values.yaml b/scripts/helmcharts/openreplay/charts/ender/values.yaml index 2d3d2b65b..c751680d4 100644 --- a/scripts/helmcharts/openreplay/charts/ender/values.yaml +++ b/scripts/helmcharts/openreplay/charts/ender/values.yaml @@ -97,3 +97,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml index 44574d1f8..eaa5d7ed1 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml @@ -88,8 +88,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/http/values.yaml b/scripts/helmcharts/openreplay/charts/http/values.yaml index 72a8acb7d..7a96d525d 100644 --- a/scripts/helmcharts/openreplay/charts/http/values.yaml +++ b/scripts/helmcharts/openreplay/charts/http/values.yaml @@ -97,3 +97,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml index 5e63e5153..e0f3fff60 100644 --- a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml @@ -62,8 +62,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/integrations/values.yaml b/scripts/helmcharts/openreplay/charts/integrations/values.yaml index b9086900b..191ed7047 100644 --- a/scripts/helmcharts/openreplay/charts/integrations/values.yaml +++ b/scripts/helmcharts/openreplay/charts/integrations/values.yaml @@ -98,3 +98,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml index 6f1a379d8..ac673fd08 100644 --- a/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml @@ -54,8 +54,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/peers/values.yaml b/scripts/helmcharts/openreplay/charts/peers/values.yaml index 721c09db9..4643a75a7 100644 --- a/scripts/helmcharts/openreplay/charts/peers/values.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/values.yaml @@ -93,3 +93,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl From eb488897f2720e5199c7e02f1c71bd79c322fb8f Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 1 Aug 2022 19:03:15 +0200 Subject: [PATCH 10/43] chore(nginx): removing ipv6, as some machines won't have support for that. Signed-off-by: rjshrjndrn --- frontend/nginx.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/nginx.conf b/frontend/nginx.conf index f5581606b..84aac6601 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,6 +1,5 @@ server { listen 8080 default_server; - listen [::]:8080 default_server; root /var/www/openreplay; index index.html; rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /index.html break; From 78ab1e49c34aa0ccc3187752e4ac625d22425c3f Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 1 Aug 2022 19:04:30 +0200 Subject: [PATCH 11/43] fix(nginx): fixing base config Signed-off-by: rjshrjndrn --- frontend/nginx.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 84aac6601..b886096da 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -2,10 +2,10 @@ server { listen 8080 default_server; root /var/www/openreplay; index index.html; - rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /index.html break; - proxy_intercept_errors on; # see frontend://nginx.org/en/docs/frontend/ngx_frontend_proxy_module.html#proxy_intercept_errors - error_page 404 =200 /index.html; location / { try_files $uri $uri/ =404; + rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /index.html break; + proxy_intercept_errors on; # see frontend://nginx.org/en/docs/frontend/ngx_frontend_proxy_module.html#proxy_intercept_errors + error_page 404 =200 /index.html; } } From 898513dd0962251d55f70cab0d08928e440751eb Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 1 Aug 2022 19:11:34 +0200 Subject: [PATCH 12/43] chore(helm): heuristics mount persistence Signed-off-by: rjshrjndrn --- .../charts/heuristics/templates/deployment.yaml | 8 ++++++++ .../openreplay/charts/heuristics/values.yaml | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml index 995e8eac2..58059f58d 100644 --- a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml @@ -60,8 +60,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/heuristics/values.yaml b/scripts/helmcharts/openreplay/charts/heuristics/values.yaml index 12d2346a3..ec8400866 100644 --- a/scripts/helmcharts/openreplay/charts/heuristics/values.yaml +++ b/scripts/helmcharts/openreplay/charts/heuristics/values.yaml @@ -97,3 +97,14 @@ nodeSelector: {} tolerations: [] affinity: {} + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl From 7a07035771098d2cb9c5e189bf7f345a8460f539 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 3 Aug 2022 19:53:00 +0200 Subject: [PATCH 13/43] feat(backend): AWS_SKIP_SSL_VALIDATION env var --- backend/Dockerfile | 1 + backend/pkg/env/aws.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/backend/Dockerfile b/backend/Dockerfile index 28bedcb40..e83ec1802 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -46,6 +46,7 @@ ENV TZ=UTC \ AWS_REGION_WEB=eu-central-1 \ AWS_REGION_IOS=eu-west-1 \ AWS_REGION_ASSETS=eu-central-1 \ + AWS_SKIP_SSL_VALIDATION=false \ CACHE_ASSETS=true \ ASSETS_SIZE_LIMIT=6291456 \ ASSETS_HEADERS="{ \"Cookie\": \"ABv=3;\" }" \ diff --git a/backend/pkg/env/aws.go b/backend/pkg/env/aws.go index cb7445797..8292cb710 100644 --- a/backend/pkg/env/aws.go +++ b/backend/pkg/env/aws.go @@ -1,7 +1,9 @@ package env import ( + "crypto/tls" "log" + "net/http" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" @@ -20,6 +22,15 @@ func AWSSessionOnRegion(region string) *_session.Session { config.Endpoint = aws.String(AWS_ENDPOINT) config.DisableSSL = aws.Bool(true) config.S3ForcePathStyle = aws.Bool(true) + + AWS_SKIP_SSL_VALIDATION := Bool("AWS_SKIP_SSL_VALIDATION") + if !AWS_SKIP_SSL_VALIDATION { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + config.HTTPClient = client + } } aws_session, err := _session.NewSession(config) if err != nil { From 54d663606016909f17b12d42dbba7b81ad81fc5c Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 3 Aug 2022 20:22:27 +0200 Subject: [PATCH 14/43] fixup! feat(backend): AWS_SKIP_SSL_VALIDATION env var --- backend/pkg/env/aws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/env/aws.go b/backend/pkg/env/aws.go index 8292cb710..e25a3a561 100644 --- a/backend/pkg/env/aws.go +++ b/backend/pkg/env/aws.go @@ -24,7 +24,7 @@ func AWSSessionOnRegion(region string) *_session.Session { config.S3ForcePathStyle = aws.Bool(true) AWS_SKIP_SSL_VALIDATION := Bool("AWS_SKIP_SSL_VALIDATION") - if !AWS_SKIP_SSL_VALIDATION { + if AWS_SKIP_SSL_VALIDATION { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } From c5208bf2c3dcfe59c17fbf7ef38df5a84c49ef6e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 4 Aug 2022 18:58:26 +0200 Subject: [PATCH 15/43] feat(chalice): SMTP allow empty username --- api/chalicelib/utils/smtp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/chalicelib/utils/smtp.py b/api/chalicelib/utils/smtp.py index 3615ca71a..403f86e0f 100644 --- a/api/chalicelib/utils/smtp.py +++ b/api/chalicelib/utils/smtp.py @@ -30,7 +30,8 @@ class SMTPClient: self.server.starttls() # stmplib docs recommend calling ehlo() before & after starttls() self.server.ehlo() - self.server.login(user=config("EMAIL_USER"), password=config("EMAIL_PASSWORD")) + if len(config("EMAIL_USER", default="")) > 0 and len(config("EMAIL_PASSWORD", default="")) > 0: + self.server.login(user=config("EMAIL_USER"), password=config("EMAIL_PASSWORD")) return self.server def __exit__(self, *args): From bd28c4ea0f01c826036f2c9184e48c94b3bb06e1 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Thu, 4 Aug 2022 22:00:48 +0200 Subject: [PATCH 16/43] fix(ui) - network request operator value --- .../components/shared/Filters/FilterOperator/FilterOperator.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/shared/Filters/FilterOperator/FilterOperator.tsx b/frontend/app/components/shared/Filters/FilterOperator/FilterOperator.tsx index 1446891e0..af977e975 100644 --- a/frontend/app/components/shared/Filters/FilterOperator/FilterOperator.tsx +++ b/frontend/app/components/shared/Filters/FilterOperator/FilterOperator.tsx @@ -79,7 +79,7 @@ function FilterOperator(props: Props) { placeholder="Select" isDisabled={isDisabled} value={value ? options.find((i: any) => i.value === value) : null} - onChange={({ value }: any) => onChange(null, { name: 'operator', value })} + onChange={({ value }: any) => onChange(null, { name: 'operator', value: value.value })} /> ); From 7b6d55a4d49beffd11c9ea4afec05ca09b65d461 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Thu, 4 Aug 2022 22:07:09 +0200 Subject: [PATCH 17/43] fix(ui) - requestbody based on project settings --- frontend/app/components/shared/SessionSearch/SessionSearch.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/shared/SessionSearch/SessionSearch.tsx b/frontend/app/components/shared/SessionSearch/SessionSearch.tsx index 66bd28a1b..bf0153057 100644 --- a/frontend/app/components/shared/SessionSearch/SessionSearch.tsx +++ b/frontend/app/components/shared/SessionSearch/SessionSearch.tsx @@ -89,6 +89,6 @@ function SessionSearch(props: Props) { } export default connect((state: any) => ({ - saveRequestPayloads: state.getIn(['site', 'active', 'saveRequestPayloads']), + saveRequestPayloads: state.getIn(['site', 'instance', 'saveRequestPayloads']), appliedFilter: state.getIn([ 'search', 'instance' ]), }), { edit, addFilter })(SessionSearch); From 4dd1431f01b4ed76cb3d8f4d49e6e788c9617c3f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 5 Aug 2022 10:58:23 +0200 Subject: [PATCH 18/43] feat(chalice): fixed unkeyed connection issue --- api/chalicelib/utils/pg_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index eda7747f8..014cfaeee 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -111,7 +111,7 @@ class PostgresClient: else: raise error finally: - if not self.long_query: + if not self.long_query and not self.unlimited_query: postgreSQL_pool.putconn(self.connection) From 7ecfd4bd5a76bed38a941cb9871d22736a33d2cb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 5 Aug 2022 17:52:16 +0200 Subject: [PATCH 19/43] feat(chalice): fixed search sessions by any error --- api/chalicelib/core/sessions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 738a5e3d9..c044a5819 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -712,13 +712,13 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event.value, value_key=e_k)) elif event_type == events.event_type.ERROR.ui_type: event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" - event.source = tuple(event.source) + event.source = list(set(event.source)) if not is_any and event.value not in [None, "*", ""]: event_where.append( _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", event.value, value_key=e_k)) if event.source[0] not in [None, "*", ""]: - event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.value, value_key=s_k)) + event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k)) # ----- IOS @@ -877,7 +877,8 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr apply = True elif f.type == schemas.FetchFilterType._duration: event_where.append( - _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, value_key=e_k_f)) + _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._request_body: event_where.append( @@ -885,7 +886,8 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr apply = True elif f.type == schemas.FetchFilterType._response_body: event_where.append( - _multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f)) + _multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, + value_key=e_k_f)) apply = True else: print(f"undefined FETCH filter: {f.type}") From 25e42fd6c533e34e7baf09c7bde44057c2b0294b Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 5 Aug 2022 19:12:10 +0200 Subject: [PATCH 20/43] fix(ui) - date range selection on switching the project --- .../app/components/BugFinder/BugFinder.js | 198 +++++++++--------- .../SessionList/SessionListHeader.js | 3 +- .../shared/Filters/FilterItem/FilterItem.tsx | 2 +- frontend/app/duck/search.js | 7 +- 4 files changed, 106 insertions(+), 104 deletions(-) diff --git a/frontend/app/components/BugFinder/BugFinder.js b/frontend/app/components/BugFinder/BugFinder.js index d3a63e49a..2c31a3ca8 100644 --- a/frontend/app/components/BugFinder/BugFinder.js +++ b/frontend/app/components/BugFinder/BugFinder.js @@ -2,14 +2,12 @@ import React from 'react'; import cn from 'classnames'; import { connect } from 'react-redux'; import withPageTitle from 'HOCs/withPageTitle'; -import { - fetchFavoriteList as fetchFavoriteSessionList -} from 'Duck/sessions'; +import { fetchFavoriteList as fetchFavoriteSessionList } from 'Duck/sessions'; import { applyFilter, clearEvents, addAttribute } from 'Duck/filters'; import { KEYS } from 'Types/filter/customFilter'; import SessionList from './SessionList'; import stl from './bugFinder.module.css'; -import withLocationHandlers from "HOCs/withLocationHandlers"; +import withLocationHandlers from 'HOCs/withLocationHandlers'; import { fetch as fetchFilterVariables } from 'Duck/sources'; import { fetchSources } from 'Duck/customField'; import { setActiveTab } from 'Duck/search'; @@ -21,113 +19,113 @@ import { clearSearch, fetchSessions, addFilterByKeyAndValue } from 'Duck/search' import { FilterKey } from 'Types/filter/filterType'; const weakEqual = (val1, val2) => { - if (!!val1 === false && !!val2 === false) return true; - if (!val1 !== !val2) return false; - return `${ val1 }` === `${ val2 }`; -} + if (!!val1 === false && !!val2 === false) return true; + if (!val1 !== !val2) return false; + return `${val1}` === `${val2}`; +}; const allowedQueryKeys = [ - 'userOs', - 'userId', - 'userBrowser', - 'userDevice', - 'userCountry', - 'startDate', - 'endDate', - 'minDuration', - 'maxDuration', - 'referrer', - 'sort', - 'order', + 'userOs', + 'userId', + 'userBrowser', + 'userDevice', + 'userCountry', + 'startDate', + 'endDate', + 'minDuration', + 'maxDuration', + 'referrer', + 'sort', + 'order', ]; @withLocationHandlers() -@connect(state => ({ - filter: state.getIn([ 'filters', 'appliedFilter' ]), - variables: state.getIn([ 'customFields', 'list' ]), - sources: state.getIn([ 'customFields', 'sources' ]), - filterValues: state.get('filterValues'), - favoriteList: state.getIn([ 'sessions', 'favoriteList' ]), - currentProjectId: state.getIn([ 'site', 'siteId' ]), - sites: state.getIn([ 'site', 'list' ]), - watchdogs: state.getIn(['watchdogs', 'list']), - activeFlow: state.getIn([ 'filters', 'activeFlow' ]), - sessions: state.getIn([ 'sessions', 'list' ]), -}), { - fetchFavoriteSessionList, - applyFilter, - addAttribute, - fetchFilterVariables, - fetchSources, - clearEvents, - setActiveTab, - clearSearch, - fetchSessions, - addFilterByKeyAndValue, -}) -@withPageTitle("Sessions - OpenReplay") +@connect( + (state) => ({ + filter: state.getIn(['filters', 'appliedFilter']), + variables: state.getIn(['customFields', 'list']), + sources: state.getIn(['customFields', 'sources']), + filterValues: state.get('filterValues'), + favoriteList: state.getIn(['sessions', 'favoriteList']), + currentProjectId: state.getIn(['site', 'siteId']), + sites: state.getIn(['site', 'list']), + watchdogs: state.getIn(['watchdogs', 'list']), + activeFlow: state.getIn(['filters', 'activeFlow']), + sessions: state.getIn(['sessions', 'list']), + }), + { + fetchFavoriteSessionList, + applyFilter, + addAttribute, + fetchFilterVariables, + fetchSources, + clearEvents, + setActiveTab, + clearSearch, + fetchSessions, + addFilterByKeyAndValue, + } +) +@withPageTitle('Sessions - OpenReplay') export default class BugFinder extends React.PureComponent { - state = {showRehydratePanel: false} - constructor(props) { - super(props); + state = { showRehydratePanel: false }; + constructor(props) { + super(props); - // TODO should cache the response - // props.fetchSources().then(() => { - // defaultFilters[6] = { - // category: 'Collaboration', - // type: 'CUSTOM', - // keys: this.props.sources.filter(({type}) => type === 'collaborationTool').map(({ label, key }) => ({ type: 'CUSTOM', source: key, label: label, key, icon: 'integrations/' + key, isFilter: false })).toJS() - // }; - // defaultFilters[7] = { - // category: 'Logging Tools', - // type: 'ERROR', - // keys: this.props.sources.filter(({type}) => type === 'logTool').map(({ label, key }) => ({ type: 'ERROR', source: key, label: label, key, icon: 'integrations/' + key, isFilter: false })).toJS() - // }; - // }); - if (props.sessions.size === 0) { - props.fetchSessions(); + // TODO should cache the response + // props.fetchSources().then(() => { + // defaultFilters[6] = { + // category: 'Collaboration', + // type: 'CUSTOM', + // keys: this.props.sources.filter(({type}) => type === 'collaborationTool').map(({ label, key }) => ({ type: 'CUSTOM', source: key, label: label, key, icon: 'integrations/' + key, isFilter: false })).toJS() + // }; + // defaultFilters[7] = { + // category: 'Logging Tools', + // type: 'ERROR', + // keys: this.props.sources.filter(({type}) => type === 'logTool').map(({ label, key }) => ({ type: 'ERROR', source: key, label: label, key, icon: 'integrations/' + key, isFilter: false })).toJS() + // }; + // }); + // if (props.sessions.size === 0) { + // props.fetchSessions(); + // } + + const queryFilter = this.props.query.all(allowedQueryKeys); + if (queryFilter.hasOwnProperty('userId')) { + props.addFilterByKeyAndValue(FilterKey.USERID, queryFilter.userId); + } else { + if (props.sessions.size === 0) { + props.fetchSessions(); + } + } } - const queryFilter = this.props.query.all(allowedQueryKeys); - if (queryFilter.hasOwnProperty('userId')) { - props.addFilterByKeyAndValue(FilterKey.USERID, queryFilter.userId); - } else { - if (props.sessions.size === 0) { - props.fetchSessions(); - } - } - } + toggleRehydratePanel = () => { + this.setState({ showRehydratePanel: !this.state.showRehydratePanel }); + }; - toggleRehydratePanel = () => { - this.setState({ showRehydratePanel: !this.state.showRehydratePanel }) - } + setActiveTab = (tab) => { + this.props.setActiveTab(tab); + }; - setActiveTab = tab => { - this.props.setActiveTab(tab); - } + render() { + const { showRehydratePanel } = this.state; - render() { - const { showRehydratePanel } = this.state; - - return ( -
-
-
- -
-
- -
- - + return ( +
+
+
+ +
+
+ +
+ + +
+ +
+
- -
-
-
- ); - } + ); + } } diff --git a/frontend/app/components/BugFinder/SessionList/SessionListHeader.js b/frontend/app/components/BugFinder/SessionList/SessionListHeader.js index 606ecaf67..afb135099 100644 --- a/frontend/app/components/BugFinder/SessionList/SessionListHeader.js +++ b/frontend/app/components/BugFinder/SessionList/SessionListHeader.js @@ -26,6 +26,7 @@ function SessionListHeader({ activeTab, count, applyFilter, filter }) { }, [label]); const { startDate, endDate, rangeValue } = filter; + console.log('startDate', startDate); const period = new Record({ start: startDate, end: endDate, rangeName: rangeValue, timezoneOffset: getTimeZoneOffset() }); const onDateChange = (e) => { @@ -40,7 +41,7 @@ function SessionListHeader({ activeTab, count, applyFilter, filter }) { const dateValues = period.toJSON(); dateValues.startDate = moment(dateValues.startDate).startOf('day').utcOffset(getTimeZoneOffset(), true).valueOf(); dateValues.endDate = moment(dateValues.endDate).endOf('day').utcOffset(getTimeZoneOffset(), true).valueOf(); - applyFilter(dateValues); + // applyFilter(dateValues); } }, [label]); diff --git a/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx b/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx index 3d82dae7c..223880b39 100644 --- a/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx +++ b/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx @@ -29,7 +29,7 @@ function FilterItem(props: Props) { }; const onOperatorChange = (e: any, { name, value }: any) => { - props.onUpdate({ ...filter, operator: value.value }); + props.onUpdate({ ...filter, operator: value }); }; const onSourceOperatorChange = (e: any, { name, value }: any) => { diff --git a/frontend/app/duck/search.js b/frontend/app/duck/search.js index 71361da14..2080e1f85 100644 --- a/frontend/app/duck/search.js +++ b/frontend/app/duck/search.js @@ -86,6 +86,8 @@ function reducer(state = initialState, action = {}) { return state.set('filterSearchList', groupedList); case APPLY_SAVED_SEARCH: return state.set('savedSearch', action.filter); + case CLEAR_SEARCH: + return state.set('savedSearch', new SavedFilter({})); case EDIT_SAVED_SEARCH: return state.mergeIn(['savedSearch'], action.instance); case UPDATE_CURRENT_PAGE: @@ -286,8 +288,9 @@ export function fetchFilterSearch(params) { } export const clearSearch = () => (dispatch, getState) => { - dispatch(applySavedSearch(new SavedFilter({}))); - dispatch(edit(new Filter({ filters: [] }))); + const filter = getState().getIn(['search', 'instance']); + // dispatch(applySavedSearch(new SavedFilter({}))); + dispatch(edit(new Filter({ startDate: filter.startDate, endDate: filter.endDate, rangeValue: filter.rangeValue, filters: [] }))); return dispatch({ type: CLEAR_SEARCH, }); From 4c65971eaff60b6fc9c34719a0c00b00974d5720 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 5 Aug 2022 19:13:04 +0200 Subject: [PATCH 21/43] change(ui) - hide fetch from filters --- frontend/app/types/filter/newFilter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/types/filter/newFilter.js b/frontend/app/types/filter/newFilter.js index 9a87da2c6..452c4f1c9 100644 --- a/frontend/app/types/filter/newFilter.js +++ b/frontend/app/types/filter/newFilter.js @@ -11,7 +11,7 @@ export const filters = [ { key: FilterKey.INPUT, type: FilterType.MULTIPLE, category: FilterCategory.INTERACTIONS, label: 'Input', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/input', isEvent: true }, { key: FilterKey.LOCATION, type: FilterType.MULTIPLE, category: FilterCategory.INTERACTIONS, label: 'Path', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/location', isEvent: true }, { key: FilterKey.CUSTOM, type: FilterType.MULTIPLE, category: FilterCategory.JAVASCRIPT, label: 'Custom Events', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/custom', isEvent: true }, - { key: FilterKey.REQUEST, type: FilterType.MULTIPLE, category: FilterCategory.JAVASCRIPT, label: 'Fetch', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/fetch', isEvent: true }, + // { key: FilterKey.REQUEST, type: FilterType.MULTIPLE, category: FilterCategory.JAVASCRIPT, label: 'Fetch', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/fetch', isEvent: true }, { key: FilterKey.FETCH, type: FilterType.SUB_FILTERS, category: FilterCategory.JAVASCRIPT, operator: 'is', label: 'Network Request', filters: [ { key: FilterKey.FETCH_URL, type: FilterType.MULTIPLE, category: FilterCategory.PERFORMANCE, label: 'with URL', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/fetch' }, { key: FilterKey.FETCH_STATUS_CODE, type: FilterType.NUMBER_MULTIPLE, category: FilterCategory.PERFORMANCE, label: 'with status code', operator: '=', operatorOptions: filterOptions.customOperators, icon: 'filters/fetch' }, From 2e009fae643adf95dc9f7852c17ace48eafc2c1b Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 5 Aug 2022 19:14:05 +0200 Subject: [PATCH 22/43] fix(ui) - fix typo --- .../components/Client/Users/components/UserForm/UserForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/Client/Users/components/UserForm/UserForm.tsx b/frontend/app/components/Client/Users/components/UserForm/UserForm.tsx index 477e42d8f..d1799cc6a 100644 --- a/frontend/app/components/Client/Users/components/UserForm/UserForm.tsx +++ b/frontend/app/components/Client/Users/components/UserForm/UserForm.tsx @@ -102,7 +102,7 @@ function UserForm(props: Props) { ) => changeCaptureRate(e.target.value)} - value={captureRate.toString()} - style={{ height: '38px', width: '100px'}} - disabled={captureAll} - min={0} - max={100} - /> - -
+ +
+ + + 100% + +
+ + {!captureAll && ( +
+ +
+ ) => changeCaptureRate(e.target.value)} + value={captureRate.toString()} + style={{ height: '38px', width: '100px' }} + disabled={captureAll} + min={0} + max={100} + /> + +
+ of the sessions @@ -80,4 +88,6 @@ function CaptureRate() { ); } -export default observer(CaptureRate); +export default connect((state: any) => ({ + isAdmin: state.getIn(['user', 'account', 'admin']) || state.getIn(['user', 'account', 'superAdmin']), +}))(observer(CaptureRate)); From 611ba855f3707d37670e9c1b03f9b58e6af213f7 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Wed, 17 Aug 2022 20:53:51 +0200 Subject: [PATCH 42/43] Details for directories under the MIT license --- LICENSE | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index b348b0c8f..a4bbf3abf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,12 +1,21 @@ Copyright (c) 2022 Asayer, Inc. OpenReplay monorepo uses multiple licenses. Portions of this software are licensed as follows: - - All content that resides under the "ee/" directory of this repository, is licensed under the license defined in "ee/LICENSE". -- Content outside of the above mentioned directories or restrictions above is available under the "Elastic License 2.0 (ELv2)" license as defined below. +- Some directories have a specific LICENSE file and are licensed under the "MIT" license, as defined below. +- Content outside of the above mentioned directories or restrictions defaults to the "Elastic License 2.0 (ELv2)" license, as defined below. Reach out (license@openreplay.com) if you have any questions regarding licenses. +------------------------------------------------------------------------------------ +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ------------------------------------------------------------------------------------ Elastic License 2.0 (ELv2) From fce2776ed905c0e1bba977a4f6259cd7c08ef659 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Wed, 17 Aug 2022 20:58:55 +0200 Subject: [PATCH 43/43] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f9f35669..31252da7d 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Check out our [roadmap](https://www.notion.so/openreplay/Roadmap-889d2c3d968b478 ## License -This repo is under the Elastic License 2.0 (ELv2), with the exception of the `ee` directory. +This monorepo uses several licenses. See [LICENSE](/LICENSE) for more details. ## Contributors