diff --git a/api/chalicelib/blueprints/bp_core.py b/api/chalicelib/blueprints/bp_core.py index 18773f68c..e99d6e297 100644 --- a/api/chalicelib/blueprints/bp_core.py +++ b/api/chalicelib/blueprints/bp_core.py @@ -897,3 +897,8 @@ def sessions_live_search(projectId, context): def get_heatmaps_by_url(projectId, context): data = app.current_request.json_body return {"data": heatmaps.get_by_url(project_id=projectId, data=data)} + + +@app.route('/general_stats', methods=['GET'], authorizer=None) +def get_general_stats(): + return {"data": {"sessions:": sessions.count_all()}} diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 340733d30..31d88ebf2 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -162,6 +162,7 @@ def search2_pg(data, project_id, user_id, favorite_only=False, errors_only=False "projectId": project_id, "userId": user_id} with pg_client.PostgresClient() as cur: + ss_constraints = [] extra_constraints = [ cur.mogrify("s.project_id = %(project_id)s", {"project_id": project_id}), cur.mogrify("s.duration IS NOT NULL", {}) @@ -173,7 +174,96 @@ def search2_pg(data, project_id, user_id, favorite_only=False, errors_only=False extra_constraints.append(cur.mogrify("fs.user_id = %(userId)s", {"userId": user_id})) events_query_part = "" + if "filters" in data: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + for f in data["filters"]: + if not isinstance(f.get("value"), list): + f["value"] = [f.get("value")] + if len(f["value"]) == 0 or f["value"][0] is None: + continue + filter_type = f["type"].upper() + f["value"] = __get_sql_value_multiple(f["value"]) + if filter_type == sessions_metas.meta_type.USERBROWSER: + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append(cur.mogrify(f's.user_browser {op} %(value)s', {"value": f["value"]})) + ss_constraints.append(cur.mogrify(f'ms.user_browser {op} %(value)s', {"value": f["value"]})) + + elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append(cur.mogrify(f's.user_os {op} %(value)s', {"value": f["value"]})) + ss_constraints.append(cur.mogrify(f'ms.user_os {op} %(value)s', {"value": f["value"]})) + + elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append(cur.mogrify(f's.user_device {op} %(value)s', {"value": f["value"]})) + ss_constraints.append(cur.mogrify(f'ms.user_device {op} %(value)s', {"value": f["value"]})) + + elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append(cur.mogrify(f's.user_country {op} %(value)s', {"value": f["value"]})) + ss_constraints.append(cur.mogrify(f'ms.user_country {op} %(value)s', {"value": f["value"]})) + elif filter_type == "duration".upper(): + if len(f["value"]) > 0 and f["value"][0] is not None: + extra_constraints.append( + cur.mogrify("s.duration >= %(minDuration)s", {"minDuration": f["value"][0]})) + ss_constraints.append( + cur.mogrify("ms.duration >= %(minDuration)s", {"minDuration": f["value"][0]})) + if len(f["value"]) > 1 and f["value"][1] is not None and f["value"][1] > 0: + extra_constraints.append( + cur.mogrify("s.duration <= %(maxDuration)s", {"maxDuration": f["value"][1]})) + ss_constraints.append( + cur.mogrify("ms.duration <= %(maxDuration)s", {"maxDuration": f["value"][1]})) + elif filter_type == sessions_metas.meta_type.REFERRER: + # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" + extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append( + cur.mogrify(f"p.base_referrer {op} %(referrer)s", {"referrer": f["value"]})) + elif filter_type == events.event_type.METADATA.ui_type: + op = __get_sql_operator(f["operator"]) + if f.get("key") in meta_keys.keys(): + extra_constraints.append( + cur.mogrify(f"s.{metadata.index_to_colname(meta_keys[f['key']])} {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)})) + ss_constraints.append( + cur.mogrify(f"ms.{metadata.index_to_colname(meta_keys[f['key']])} {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)})) + elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + op = __get_sql_operator(f["operator"]) + extra_constraints.append( + cur.mogrify(f"s.user_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + ss_constraints.append( + cur.mogrify(f"ms.user_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, + sessions_metas.meta_type.USERANONYMOUSID_IOS]: + op = __get_sql_operator(f["operator"]) + extra_constraints.append( + cur.mogrify(f"s.user_anonymous_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + ss_constraints.append( + cur.mogrify(f"ms.user_anonymous_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + op = __get_sql_operator(f["operator"]) + extra_constraints.append( + cur.mogrify(f"s.rev_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + ss_constraints.append( + cur.mogrify(f"ms.rev_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + + # --------------------------------------------------------------------------- if len(data.get("events", [])) > 0: + ss_constraints = [s.decode('UTF-8') for s in ss_constraints] events_query_from = [] event_index = 0 @@ -268,7 +358,8 @@ def search2_pg(data, project_id, user_id, favorite_only=False, errors_only=False else: continue - + if event_index == 0: + event_where += ss_constraints if is_not: if event_index == 0: events_query_from.append(cur.mogrify(f"""\ @@ -316,73 +407,6 @@ def search2_pg(data, project_id, user_id, favorite_only=False, errors_only=False else: data["events"] = [] - # --------------------------------------------------------------------------- - if "filters" in data: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} - for f in data["filters"]: - if not isinstance(f.get("value"), list): - f["value"] = [f.get("value")] - if len(f["value"]) == 0 or f["value"][0] is None: - continue - filter_type = f["type"].upper() - f["value"] = __get_sql_value_multiple(f["value"]) - if filter_type == sessions_metas.meta_type.USERBROWSER: - op = __get_sql_operator_multiple(f["operator"]) - extra_constraints.append( - cur.mogrify(f's.user_browser {op} %(value)s', {"value": f["value"]})) - - elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: - op = __get_sql_operator_multiple(f["operator"]) - extra_constraints.append(cur.mogrify(f's.user_os {op} %(value)s', {"value": f["value"]})) - - elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: - op = __get_sql_operator_multiple(f["operator"]) - extra_constraints.append(cur.mogrify(f's.user_device {op} %(value)s', {"value": f["value"]})) - - elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: - op = __get_sql_operator_multiple(f["operator"]) - extra_constraints.append(cur.mogrify(f's.user_country {op} %(value)s', {"value": f["value"]})) - elif filter_type == "duration".upper(): - if len(f["value"]) > 0 and f["value"][0] is not None: - extra_constraints.append( - cur.mogrify("s.duration >= %(minDuration)s", {"minDuration": f["value"][0]})) - if len(f["value"]) > 1 and f["value"][1] is not None and f["value"][1] > 0: - extra_constraints.append( - cur.mogrify("s.duration <= %(maxDuration)s", {"maxDuration": f["value"][1]})) - elif filter_type == sessions_metas.meta_type.REFERRER: - # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" - extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" - op = __get_sql_operator_multiple(f["operator"]) - extra_constraints.append( - cur.mogrify(f"p.base_referrer {op} %(referrer)s", {"referrer": f["value"]})) - elif filter_type == events.event_type.METADATA.ui_type: - op = __get_sql_operator(f["operator"]) - if f.get("key") in meta_keys.keys(): - extra_constraints.append( - cur.mogrify(f"s.{metadata.index_to_colname(meta_keys[f['key']])} {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) - ) - elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: - op = __get_sql_operator(f["operator"]) - extra_constraints.append( - cur.mogrify(f"s.user_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) - ) - elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, - sessions_metas.meta_type.USERANONYMOUSID_IOS]: - op = __get_sql_operator(f["operator"]) - extra_constraints.append( - cur.mogrify(f"s.user_anonymous_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) - ) - elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: - op = __get_sql_operator(f["operator"]) - extra_constraints.append( - cur.mogrify(f"s.rev_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) - ) - # --------------------------------------------------------------------------- if data.get("startDate") is not None: @@ -741,3 +765,9 @@ def delete_sessions_by_user_ids(project_id, user_ids): cur.execute(query=query) return True + + +def count_all(): + with pg_client.PostgresClient() as cur: + row = cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") + return row.get("count", 0) diff --git a/backend/services/http/handlers.go b/backend/services/http/handlers.go index 81cd6e9c7..2ac2852a2 100644 --- a/backend/services/http/handlers.go +++ b/backend/services/http/handlers.go @@ -14,8 +14,8 @@ import ( gzip "github.com/klauspost/pgzip" "openreplay/backend/pkg/db/postgres" - . "openreplay/backend/pkg/messages" "openreplay/backend/pkg/token" + . "openreplay/backend/pkg/messages" ) const JSON_SIZE_LIMIT int64 = 1e3 // 1Kb @@ -124,17 +124,22 @@ func pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64) { body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) //defer body.Close() var reader io.ReadCloser + var err error switch r.Header.Get("Content-Encoding") { case "gzip": - reader, err := gzip.NewReader(body) + log.Println("Gzip", reader) + + reader, err = gzip.NewReader(body) if err != nil { responseWithError(w, http.StatusInternalServerError, err) // TODO: stage-dependent responce return } + log.Println("Gzip reader init", reader) defer reader.Close() default: reader = body } + log.Println("Reader after switch:", reader) buf, err := ioutil.ReadAll(reader) if err != nil { responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging diff --git a/backend/services/http/handlers_ios.go b/backend/services/http/handlers_ios.go index 32f4a271a..fd30eb5c2 100644 --- a/backend/services/http/handlers_ios.go +++ b/backend/services/http/handlers_ios.go @@ -1,145 +1,180 @@ package main -// const FILES_SIZE_LIMIT int64 = 1e8 // 100Mb +import ( + "encoding/json" + "net/http" + "errors" + "time" + "math/rand" + "strconv" -// func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { -// type request struct { -// // SessionID *string -// EncodedProjectID *uint64 `json:"projectID"` -// TrackerVersion string `json:"trackerVersion"` -// RevID string `json:"revID"` -// UserUUID *string `json:"userUUID"` -// //UserOS string `json"userOS"` //hardcoded 'MacOS' -// UserOSVersion string `json:"userOSVersion"` -// UserDevice string `json:"userDevice"` -// Timestamp uint64 `json:"timestamp"` -// // UserDeviceType uint 0:phone 1:pad 2:tv 3:carPlay 5:mac -// // “performances”:{ -// // “activeProcessorCount”:8, -// // “isLowPowerModeEnabled”:0, -// // “orientation”:0, -// // “systemUptime”:585430, -// // “batteryState”:0, -// // “thermalState”:0, -// // “batteryLevel”:0, -// // “processorCount”:8, -// // “physicalMemory”:17179869184 -// // }, -// } -// type response struct { -// Token string `json:"token"` -// ImagesHashList []string `json:"imagesHashList"` -// UserUUID string `json:"userUUID"` -// SESSION_ID uint64 `json:"SESSION_ID"` ///TEMP -// } -// startTime := time.Now() -// req := &request{} -// body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) -// //defer body.Close() -// if err := json.NewDecoder(body).Decode(req); err != nil { -// responseWithError(w, http.StatusBadRequest, err) -// return -// } + "openreplay/backend/pkg/db/postgres" + "openreplay/backend/pkg/token" + . "openreplay/backend/pkg/messages" +) -// if req.EncodedProjectID == nil { -// responseWithError(w, http.StatusForbidden, errors.New("ProjectID value required")) -// return -// } -// projectID := decodeProjectID(*(req.EncodedProjectID)) -// if projectID == 0 { -// responseWithError(w, http.StatusUnprocessableEntity, errors.New("ProjectID value is invalid")) -// return -// } -// p, err := pgconn.GetProject(uint32(projectID)) -// if err != nil { -// if postgres.IsNoRowsErr(err) { -// responseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active")) -// } else { -// responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging -// } -// return -// } -// sessionID, err := flaker.Compose(req.Timestamp) -// if err != nil { -// responseWithError(w, http.StatusInternalServerError, err) -// return -// } -// userUUID := getUUID(req.UserUUID) -// country := geoIP.ExtractISOCodeFromHTTPRequest(r) -// expirationTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) +const FILES_SIZE_LIMIT int64 = 1e8 // 100Mb -// imagesHashList, err := s3.GetFrequentlyUsedKeys(*(req.EncodedProjectID)) // TODO: reuse index: ~ frequency * size -// if err != nil { -// responseWithError(w, http.StatusInternalServerError, err) -// return -// } +func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { + type request struct { + Token string `json:"token"` + ProjectKey *string `json:"projectKey"` + TrackerVersion string `json:"trackerVersion"` + RevID string `json:"revID"` + UserUUID *string `json:"userUUID"` + //UserOS string `json"userOS"` //hardcoded 'MacOS' + UserOSVersion string `json:"userOSVersion"` + UserDevice string `json:"userDevice"` + Timestamp uint64 `json:"timestamp"` + // UserDeviceType uint 0:phone 1:pad 2:tv 3:carPlay 5:mac + // “performances”:{ + // “activeProcessorCount”:8, + // “isLowPowerModeEnabled”:0, + // “orientation”:0, + // “systemUptime”:585430, + // “batteryState”:0, + // “thermalState”:0, + // “batteryLevel”:0, + // “processorCount”:8, + // “physicalMemory”:17179869184 + // }, + } + type response struct { + Token string `json:"token"` + ImagesHashList []string `json:"imagesHashList"` + UserUUID string `json:"userUUID"` + BeaconSizeLimit int64 `json:"beaconSizeLimit"` + SessionID string `json:"sessionID"` + } + startTime := time.Now() + req := &request{} + body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) + //defer body.Close() + if err := json.NewDecoder(body).Decode(req); err != nil { + responseWithError(w, http.StatusBadRequest, err) + return + } -// responseWithJSON(w, &response{ -// Token: tokenizer.Compose(sessionID, uint64(expirationTime.UnixNano()/1e6)), -// ImagesHashList: imagesHashList, -// UserUUID: userUUID, -// //TEMP: -// SESSION_ID: sessionID, -// }) -// producer.Produce(topicRaw, sessionID, messages.Encode(&messages.IOSSessionStart{ -// Timestamp: req.Timestamp, -// ProjectID: projectID, -// TrackerVersion: req.TrackerVersion, -// RevID: req.RevID, -// UserUUID: userUUID, -// UserOS: "MacOS", -// UserOSVersion: req.UserOSVersion, -// UserDevice: MapIOSDevice(req.UserDevice), -// UserDeviceType: GetIOSDeviceType(req.UserDevice), // string `json:"userDeviceType"` // From UserDevice; ENUM ? -// UserCountry: country, -// })) -// } + if req.ProjectKey == nil { + responseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) + return + } + + p, err := pgconn.GetProjectByKey(*req.ProjectKey) + if err != nil { + if postgres.IsNoRowsErr(err) { + responseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active")) + } else { + responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + } + return + } + userUUID := getUUID(req.UserUUID) + tokenData, err := tokenizer.Parse(req.Token) + + if err != nil { // Starting the new one + dice := byte(rand.Intn(100)) // [0, 100) + if dice >= p.SampleRate { + responseWithError(w, http.StatusForbidden, errors.New("cancel")) + return + } + + ua := uaParser.ParseFromHTTPRequest(r) + if ua == nil { + responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) + return + } + sessionID, err := flaker.Compose(uint64(startTime.UnixNano() / 1e6)) + if err != nil { + responseWithError(w, http.StatusInternalServerError, err) + return + } + // TODO: if EXPIRED => send message for two sessions association + expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) + tokenData = &token.TokenData{sessionID, expTime.UnixNano() / 1e6} + + country := geoIP.ExtractISOCodeFromHTTPRequest(r) + + // The difference with web is mostly here: + producer.Produce(TOPIC_RAW, tokenData.ID, Encode(&IOSSessionStart{ + Timestamp: req.Timestamp, + ProjectID: uint64(p.ProjectID), + TrackerVersion: req.TrackerVersion, + RevID: req.RevID, + UserUUID: userUUID, + UserOS: "IOS", + UserOSVersion: req.UserOSVersion, + UserDevice: MapIOSDevice(req.UserDevice), + UserDeviceType: GetIOSDeviceType(req.UserDevice), + UserCountry: country, + })) + } + + // imagesHashList, err := s3.GetFrequentlyUsedKeys(*(req.EncodedProjectID)) // TODO: reuse index: ~ frequency * size + // if err != nil { + // responseWithError(w, http.StatusInternalServerError, err) + // return + // } + + responseWithJSON(w, &response{ + // ImagesHashList: imagesHashList, + Token: tokenizer.Compose(*tokenData), + UserUUID: userUUID, + SessionID: strconv.FormatUint(tokenData.ID, 10), + BeaconSizeLimit: BEACON_SIZE_LIMIT, + }) +} -// func pushLateMessagesHandler(w http.ResponseWriter, r *http.Request) { -// sessionData, err := tokenizer.ParseFromHTTPRequest(r) -// if err != nil && err != token.EXPIRED { -// responseWithError(w, http.StatusUnauthorized, err) -// return -// } -// // Check timestamps here? -// pushMessages(w, r, sessionData.ID) -// } +func pushLateMessagesHandler(w http.ResponseWriter, r *http.Request) { + sessionData, err := tokenizer.ParseFromHTTPRequest(r) + if err != nil && err != token.EXPIRED { + responseWithError(w, http.StatusUnauthorized, err) + return + } + // Check timestamps here? + pushMessages(w, r, sessionData.ID) +} -// func iosImagesUploadHandler(w http.ResponseWriter, r *http.Request) { -// r.Body = http.MaxBytesReader(w, r.Body, FILES_SIZE_LIMIT) -// // defer r.Body.Close() -// err := r.ParseMultipartForm(1e5) // 100Kb -// if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { -// responseWithError(w, http.StatusUnsupportedMediaType, err) -// // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB -// } else if err != nil { -// responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging -// } +func iosImagesUploadHandler(w http.ResponseWriter, r *http.Request) { + sessionData, err := tokenizer.ParseFromHTTPRequest(r) + if err != nil { // Should accept expired token? + responseWithError(w, http.StatusUnauthorized, err) + return + } -// if len(r.MultipartForm.Value["projectID"]) == 0 { -// responseWithError(w, http.StatusBadRequest, errors.New("projectID parameter required")) // status for missing/wrong parameter? -// return -// } -// // encodedProjectID, err := strconv.ParseUint(r.MultipartForm.Value["projectID"][0], 10, 64) -// // projectID := decodeProjectID(encodedProjectID) -// // if projectID == 0 || err != nil { -// // responseWithError(w, http.StatusUnprocessableEntity, errors.New("projectID value is incorrect")) -// // return -// // } -// prefix := r.MultipartForm.Value["projectID"][0] + "/" //strconv.FormatUint(uint64(projectID), 10) + "/" + r.Body = http.MaxBytesReader(w, r.Body, FILES_SIZE_LIMIT) + // defer r.Body.Close() + err = r.ParseMultipartForm(1e6) // ~1Mb + if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { + responseWithError(w, http.StatusUnsupportedMediaType, err) + // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB + } else if err != nil { + responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + } -// for _, fileHeaderList := range r.MultipartForm.File { -// for _, fileHeader := range fileHeaderList { -// file, err := fileHeader.Open() -// if err != nil { -// continue // TODO: send server error or accumulate successful files -// } -// key := prefix + fileHeader.Filename // TODO: Malicious image put: use jwt? -// go s3.Upload(file, key, "image/png", false) -// } -// } + if (r.MultipartForm == nil) { + responseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) + } -// w.WriteHeader(http.StatusOK) -// } + if len(r.MultipartForm.Value["projectKey"]) == 0 { + responseWithError(w, http.StatusBadRequest, errors.New("projectKey parameter missing")) // status for missing/wrong parameter? + return + } + + prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/" + + for _, fileHeaderList := range r.MultipartForm.File { + for _, fileHeader := range fileHeaderList { + file, err := fileHeader.Open() + if err != nil { + continue // TODO: send server error or accumulate successful files + } + key := prefix + fileHeader.Filename + go s3.Upload(file, key, "image/png", false) + } + } + + w.WriteHeader(http.StatusOK) +} diff --git a/backend/services/http/main.go b/backend/services/http/main.go index dc2eb1720..cac5a2842 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -76,6 +76,9 @@ func main() { return } + log.Printf("Request: %v - %v ", r.Method, r.URL.Path) + + switch r.URL.Path { case "/": w.WriteHeader(http.StatusOK) @@ -100,34 +103,34 @@ func main() { default: w.WriteHeader(http.StatusMethodNotAllowed) } - // case "/v1/ios/start": - // switch r.Method { - // case http.MethodPost: - // startSessionHandlerIOS(w, r) - // default: - // w.WriteHeader(http.StatusMethodNotAllowed) - // } - // case "/v1/ios/append": - // switch r.Method { - // case http.MethodPost: - // pushMessagesHandler(w, r) - // default: - // w.WriteHeader(http.StatusMethodNotAllowed) - // } - // case "/v1/ios/late": - // switch r.Method { - // case http.MethodPost: - // pushLateMessagesHandler(w, r) - // default: - // w.WriteHeader(http.StatusMethodNotAllowed) - // } - // case "/v1/ios/images": - // switch r.Method { - // case http.MethodPost: - // iosImagesUploadHandler(w, r) - // default: - // w.WriteHeader(http.StatusMethodNotAllowed) - // } + case "/v1/ios/start": + switch r.Method { + case http.MethodPost: + startSessionHandlerIOS(w, r) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + case "/v1/ios/i": + switch r.Method { + case http.MethodPost: + pushMessagesHandler(w, r) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + case "/v1/ios/late": + switch r.Method { + case http.MethodPost: + pushLateMessagesHandler(w, r) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + case "/v1/ios/images": + switch r.Method { + case http.MethodPost: + iosImagesUploadHandler(w, r) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } default: w.WriteHeader(http.StatusNotFound) } diff --git a/backend/services/http/response.go b/backend/services/http/response.go index ffd22875e..11d9b328d 100644 --- a/backend/services/http/response.go +++ b/backend/services/http/response.go @@ -11,6 +11,7 @@ func responseWithJSON(w http.ResponseWriter, res interface{}) { if err != nil { log.Println(err) } + w.Header().Set("Content-Type", "application/json") w.Write(body) } diff --git a/frontend/app/components/Session_/Storage/Storage.js b/frontend/app/components/Session_/Storage/Storage.js index db6d9216b..27162b5da 100644 --- a/frontend/app/components/Session_/Storage/Storage.js +++ b/frontend/app/components/Session_/Storage/Storage.js @@ -26,7 +26,7 @@ import stl from './storage.css'; function getActionsName(type) { switch(type) { case STORAGE_TYPES.MOBX: - return "EVENTS"; + return "MUTATIONS"; case STORAGE_TYPES.VUEX: return "MUTATIONS"; default: @@ -141,7 +141,7 @@ export default class Storage extends React.PureComponent { break; case STORAGE_TYPES.MOBX: src = item.payload; - name = `@${item.type} ${src && src.name}`; + name = `@${item.type} ${src && src.type}`; break; } diff --git a/scripts/helm/app/openreplay/templates/deployment.yaml b/scripts/helm/app/openreplay/templates/deployment.yaml index 187025b74..da9963fe7 100644 --- a/scripts/helm/app/openreplay/templates/deployment.yaml +++ b/scripts/helm/app/openreplay/templates/deployment.yaml @@ -50,9 +50,9 @@ spec: {{- if eq .Values.pvc.name "hostPath" }} volumeMounts: - mountPath: {{ .Values.pvc.mountPath }} - name: {{ .Values.pvc.name }} + name: datadir volumes: - - name: mydir + - name: datadir hostPath: # Ensure the file directory is created. path: {{ .Values.pvc.hostMountPath }} diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 056e6b8c2..8458199cb 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1,8 +1,8 @@ BEGIN; --- --- public.sql --- +-- Schemas and functions definitions: +CREATE SCHEMA IF NOT EXISTS events_common; +CREATE SCHEMA IF NOT EXISTS events; -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -- --- accounts.sql --- CREATE OR REPLACE FUNCTION generate_api_key(length integer) RETURNS text AS @@ -23,802 +23,7 @@ begin end; $$ LANGUAGE plpgsql; - - -CREATE TABLE public.tenants -( - tenant_id integer NOT NULL DEFAULT 1, - user_id text NOT NULL DEFAULT generate_api_key(20), - name text NOT NULL, - api_key text NOT NULL DEFAULT generate_api_key(20), - created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), - edition varchar(3) NOT NULL, - version_number text NOT NULL, - license text NULL, - opt_out bool NOT NULL DEFAULT FALSE, - t_projects integer NOT NULL DEFAULT 1, - t_sessions bigint NOT NULL DEFAULT 0, - t_users integer NOT NULL DEFAULT 1, - t_integrations integer NOT NULL DEFAULT 0, - CONSTRAINT onerow_uni CHECK (tenant_id = 1) -); - -CREATE TYPE user_role AS ENUM ('owner', 'admin', 'member'); - -CREATE TABLE users -( - user_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - email text NOT NULL UNIQUE, - role user_role NOT NULL DEFAULT 'member', - name text NOT NULL, - created_at timestamp without time zone NOT NULL default (now() at time zone 'utc'), - deleted_at timestamp without time zone NULL DEFAULT NULL, - appearance jsonb NOT NULL default '{ - "role": "dev", - "dashboard": { - "cpu": true, - "fps": false, - "avgCpu": true, - "avgFps": true, - "errors": true, - "crashes": true, - "overview": true, - "sessions": true, - "topMetrics": true, - "callsErrors": true, - "pageMetrics": true, - "performance": true, - "timeToRender": false, - "userActivity": false, - "avgFirstPaint": false, - "countSessions": true, - "errorsPerType": true, - "slowestImages": true, - "speedLocation": true, - "slowestDomains": true, - "avgPageLoadTime": true, - "avgTillFirstBit": false, - "avgTimeToRender": true, - "avgVisitedPages": false, - "avgImageLoadTime": true, - "busiestTimeOfDay": true, - "errorsPerDomains": true, - "missingResources": true, - "resourcesByParty": true, - "sessionsFeedback": false, - "slowestResources": true, - "avgUsedJsHeapSize": true, - "domainsErrors_4xx": true, - "domainsErrors_5xx": true, - "memoryConsumption": true, - "pagesDomBuildtime": false, - "pagesResponseTime": true, - "avgRequestLoadTime": true, - "avgSessionDuration": false, - "sessionsPerBrowser": false, - "applicationActivity": true, - "sessionsFrustration": false, - "avgPagesDomBuildtime": true, - "avgPagesResponseTime": false, - "avgTimeToInteractive": true, - "resourcesCountByType": true, - "resourcesLoadingTime": true, - "avgDomContentLoadStart": true, - "avgFirstContentfulPixel": false, - "resourceTypeVsResponseEnd": true, - "impactedSessionsByJsErrors": true, - "impactedSessionsBySlowPages": true, - "resourcesVsVisuallyComplete": true, - "pagesResponseTimeDistribution": true - }, - "sessionsLive": false, - "sessionsDevtools": true - }'::jsonb, - api_key text UNIQUE default generate_api_key(20) not null, - jwt_iat timestamp without time zone NULL DEFAULT NULL, - data jsonb NOT NULL DEFAULT '{}'::jsonb, - weekly_report boolean NOT NULL DEFAULT TRUE -); - -CREATE TABLE basic_authentication -( - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, - password text DEFAULT NULL, - generated_password boolean NOT NULL DEFAULT false, - invitation_token text NULL DEFAULT NULL, - invited_at timestamp without time zone NULL DEFAULT NULL, - change_pwd_token text NULL DEFAULT NULL, - change_pwd_expire_at timestamp without time zone NULL DEFAULT NULL, - changed_at timestamp, - UNIQUE (user_id) -); - -CREATE TYPE oauth_provider AS ENUM ('jira', 'github'); -CREATE TABLE oauth_authentication -( - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, - provider oauth_provider NOT NULL, - provider_user_id text NOT NULL, - token text NOT NULL, - UNIQUE (user_id, provider) -); - --- --- projects.sql --- - -CREATE TABLE projects -( - project_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_key varchar(20) NOT NULL UNIQUE DEFAULT generate_api_key(20), - name text NOT NULL, - active boolean NOT NULL, - sample_rate smallint NOT NULL DEFAULT 100 CHECK (sample_rate >= 0 AND sample_rate <= 100), - created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), - deleted_at timestamp without time zone NULL DEFAULT NULL, - max_session_duration integer NOT NULL DEFAULT 7200000, - metadata_1 text DEFAULT NULL, - metadata_2 text DEFAULT NULL, - metadata_3 text DEFAULT NULL, - metadata_4 text DEFAULT NULL, - metadata_5 text DEFAULT NULL, - metadata_6 text DEFAULT NULL, - metadata_7 text DEFAULT NULL, - metadata_8 text DEFAULT NULL, - metadata_9 text DEFAULT NULL, - metadata_10 text DEFAULT NULL, - gdpr jsonb NOT NULL DEFAULT '{ - "maskEmails": true, - "sampleRate": 33, - "maskNumbers": false, - "defaultInputMode": "plain" - }'::jsonb -- ?????? -); - -CREATE OR REPLACE FUNCTION notify_project() RETURNS trigger AS -$$ -BEGIN - PERFORM pg_notify('project', row_to_json(NEW)::text); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER on_insert_or_update - AFTER INSERT OR UPDATE - ON projects - FOR EACH ROW -EXECUTE PROCEDURE notify_project(); - --- --- alerts.sql --- - -CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); - -CREATE TABLE alerts -( - alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - name text NOT NULL, - description text NULL DEFAULT NULL, - active boolean NOT NULL DEFAULT TRUE, - detection_method alert_detection_method NOT NULL, - query jsonb NOT NULL, - deleted_at timestamp NULL DEFAULT NULL, - created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - options jsonb NOT NULL DEFAULT '{ - "renotifyInterval": 1440 - }'::jsonb -); - - -CREATE OR REPLACE FUNCTION notify_alert() RETURNS trigger AS -$$ -DECLARE - clone jsonb; -BEGIN - clone = to_jsonb(NEW); - clone = jsonb_set(clone, '{created_at}', to_jsonb(CAST(EXTRACT(epoch FROM NEW.created_at) * 1000 AS BIGINT))); - IF NEW.deleted_at NOTNULL THEN - clone = jsonb_set(clone, '{deleted_at}', to_jsonb(CAST(EXTRACT(epoch FROM NEW.deleted_at) * 1000 AS BIGINT))); - END IF; - PERFORM pg_notify('alert', clone::text); - RETURN NEW; -END ; -$$ LANGUAGE plpgsql; - - -CREATE TRIGGER on_insert_or_update_or_delete - AFTER INSERT OR UPDATE OR DELETE - ON alerts - FOR EACH ROW -EXECUTE PROCEDURE notify_alert(); - --- --- webhooks.sql --- - -create type webhook_type as enum ('webhook', 'slack', 'email'); - -create table webhooks -( - webhook_id integer generated by default as identity - constraint webhooks_pkey - primary key, - endpoint text not null, - created_at timestamp default timezone('utc'::text, now()) not null, - deleted_at timestamp, - auth_header text, - type webhook_type not null, - index integer default 0 not null, - name varchar(100) -); - --- --- notifications.sql --- - -CREATE TABLE notifications -( - notification_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - user_id integer REFERENCES users (user_id) ON DELETE CASCADE, - title text NOT NULL, - description text NOT NULL, - button_text varchar(80) NULL, - button_url text NULL, - image_url text NULL, - created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - options jsonb NOT NULL DEFAULT '{}'::jsonb -); - -CREATE INDEX notifications_user_id_index ON notifications (user_id); -CREATE INDEX notifications_created_at_index ON notifications (created_at DESC); -CREATE INDEX notifications_created_at_epoch_idx ON notifications (CAST(EXTRACT(EPOCH FROM created_at) * 1000 AS BIGINT) DESC); - -CREATE TABLE user_viewed_notifications -( - user_id integer NOT NULL REFERENCES users (user_id) on delete cascade, - notification_id integer NOT NULL REFERENCES notifications (notification_id) on delete cascade, - constraint user_viewed_notifications_pkey primary key (user_id, notification_id) -); - --- --- funnels.sql --- - -CREATE TABLE funnels -( - funnel_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, - name text not null, - filter jsonb not null, - created_at timestamp default timezone('utc'::text, now()) not null, - deleted_at timestamp, - is_public boolean NOT NULL DEFAULT False -); - -CREATE INDEX ON public.funnels (user_id, is_public); - --- --- announcements.sql --- - -create type announcement_type as enum ('notification', 'alert'); - -create table announcements -( - announcement_id serial not null - constraint announcements_pk - primary key, - title text not null, - description text not null, - button_text varchar(30), - button_url text, - image_url text, - created_at timestamp default timezone('utc'::text, now()) not null, - type announcement_type default 'notification'::announcement_type not null -); - --- --- integrations.sql --- - -CREATE TYPE integration_provider AS ENUM ('bugsnag', 'cloudwatch', 'datadog', 'newrelic', 'rollbar', 'sentry', 'stackdriver', 'sumologic', 'elasticsearch'); --, 'jira', 'github'); -CREATE TABLE integrations -( - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - provider integration_provider NOT NULL, - options jsonb NOT NULL, - request_data jsonb NOT NULL DEFAULT '{}'::jsonb, - PRIMARY KEY (project_id, provider) -); - -CREATE OR REPLACE FUNCTION notify_integration() RETURNS trigger AS -$$ -BEGIN - IF NEW IS NULL THEN - PERFORM pg_notify('integration', (row_to_json(OLD)::text || '{"options": null, "request_data": null}'::text)); - ELSIF (OLD IS NULL) OR (OLD.options <> NEW.options) THEN - PERFORM pg_notify('integration', row_to_json(NEW)::text); - END IF; - RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER on_insert_or_update_or_delete - AFTER INSERT OR UPDATE OR DELETE - ON integrations - FOR EACH ROW -EXECUTE PROCEDURE notify_integration(); - - -create table jira_cloud -( - user_id integer not null - constraint jira_cloud_pk - primary key - constraint jira_cloud_users_fkey - references users - on delete cascade, - username text not null, - token text not null, - url text -); - --- --- issues.sql --- - -CREATE TYPE issue_type AS ENUM ( - 'click_rage', - 'dead_click', - 'excessive_scrolling', - 'bad_request', - 'missing_resource', - 'memory', - 'cpu', - 'slow_resource', - 'slow_page_load', - 'crash', - 'ml_cpu', - 'ml_memory', - 'ml_dead_click', - 'ml_click_rage', - 'ml_mouse_thrashing', - 'ml_excessive_scrolling', - 'ml_slow_resources', - 'custom', - 'js_exception' - ); - -CREATE TABLE issues -( - issue_id text NOT NULL PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - type issue_type NOT NULL, - context_string text NOT NULL, - context jsonb DEFAULT NULL -); -CREATE INDEX ON issues (issue_id, type); -CREATE INDEX issues_context_string_gin_idx ON public.issues USING GIN (context_string gin_trgm_ops); -CREATE INDEX issues_project_id_idx ON issues (project_id); - --- --- errors.sql --- - -CREATE TYPE error_source AS ENUM ('js_exception', 'bugsnag', 'cloudwatch', 'datadog', 'newrelic', 'rollbar', 'sentry', 'stackdriver', 'sumologic'); -CREATE TYPE error_status AS ENUM ('unresolved', 'resolved', 'ignored'); -CREATE TABLE errors -( - error_id text NOT NULL PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - source error_source NOT NULL, - name text DEFAULT NULL, - message text NOT NULL, - payload jsonb NOT NULL, - status error_status NOT NULL DEFAULT 'unresolved', - parent_error_id text DEFAULT NULL REFERENCES errors (error_id) ON DELETE SET NULL, - stacktrace jsonb, --to save the stacktrace and not query S3 another time - stacktrace_parsed_at timestamp -); -CREATE INDEX errors_error_id_idx ON errors (error_id); -CREATE INDEX ON errors (project_id, source); -CREATE INDEX errors_message_gin_idx ON public.errors USING GIN (message gin_trgm_ops); -CREATE INDEX errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops); -CREATE INDEX errors_project_id_idx ON public.errors (project_id); -CREATE INDEX errors_project_id_status_idx ON public.errors (project_id, status); -CREATE INDEX errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception'; -CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id); -CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; - -CREATE TABLE user_favorite_errors -( - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, - error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, - PRIMARY KEY (user_id, error_id) -); - -CREATE TABLE user_viewed_errors -( - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, - error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, - PRIMARY KEY (user_id, error_id) -); -CREATE INDEX user_viewed_errors_user_id_idx ON public.user_viewed_errors (user_id); -CREATE INDEX user_viewed_errors_error_id_idx ON public.user_viewed_errors (error_id); - - --- --- sessions.sql --- -CREATE TYPE device_type AS ENUM ('desktop', 'tablet', 'mobile', 'other'); -CREATE TYPE country AS ENUM ('UN', 'RW', 'SO', 'YE', 'IQ', 'SA', 'IR', 'CY', 'TZ', 'SY', 'AM', 'KE', 'CD', 'DJ', 'UG', 'CF', 'SC', 'JO', 'LB', 'KW', 'OM', 'QA', 'BH', 'AE', 'IL', 'TR', 'ET', 'ER', 'EG', 'SD', 'GR', 'BI', 'EE', 'LV', 'AZ', 'LT', 'SJ', 'GE', 'MD', 'BY', 'FI', 'AX', 'UA', 'MK', 'HU', 'BG', 'AL', 'PL', 'RO', 'XK', 'ZW', 'ZM', 'KM', 'MW', 'LS', 'BW', 'MU', 'SZ', 'RE', 'ZA', 'YT', 'MZ', 'MG', 'AF', 'PK', 'BD', 'TM', 'TJ', 'LK', 'BT', 'IN', 'MV', 'IO', 'NP', 'MM', 'UZ', 'KZ', 'KG', 'TF', 'HM', 'CC', 'PW', 'VN', 'TH', 'ID', 'LA', 'TW', 'PH', 'MY', 'CN', 'HK', 'BN', 'MO', 'KH', 'KR', 'JP', 'KP', 'SG', 'CK', 'TL', 'RU', 'MN', 'AU', 'CX', 'MH', 'FM', 'PG', 'SB', 'TV', 'NR', 'VU', 'NC', 'NF', 'NZ', 'FJ', 'LY', 'CM', 'SN', 'CG', 'PT', 'LR', 'CI', 'GH', 'GQ', 'NG', 'BF', 'TG', 'GW', 'MR', 'BJ', 'GA', 'SL', 'ST', 'GI', 'GM', 'GN', 'TD', 'NE', 'ML', 'EH', 'TN', 'ES', 'MA', 'MT', 'DZ', 'FO', 'DK', 'IS', 'GB', 'CH', 'SE', 'NL', 'AT', 'BE', 'DE', 'LU', 'IE', 'MC', 'FR', 'AD', 'LI', 'JE', 'IM', 'GG', 'SK', 'CZ', 'NO', 'VA', 'SM', 'IT', 'SI', 'ME', 'HR', 'BA', 'AO', 'NA', 'SH', 'BV', 'BB', 'CV', 'GY', 'GF', 'SR', 'PM', 'GL', 'PY', 'UY', 'BR', 'FK', 'GS', 'JM', 'DO', 'CU', 'MQ', 'BS', 'BM', 'AI', 'TT', 'KN', 'DM', 'AG', 'LC', 'TC', 'AW', 'VG', 'VC', 'MS', 'MF', 'BL', 'GP', 'GD', 'KY', 'BZ', 'SV', 'GT', 'HN', 'NI', 'CR', 'VE', 'EC', 'CO', 'PA', 'HT', 'AR', 'CL', 'BO', 'PE', 'MX', 'PF', 'PN', 'KI', 'TK', 'TO', 'WF', 'WS', 'NU', 'MP', 'GU', 'PR', 'VI', 'UM', 'AS', 'CA', 'US', 'PS', 'RS', 'AQ', 'SX', 'CW', 'BQ', 'SS'); -CREATE TYPE platform AS ENUM ('web','ios','android'); - -CREATE TABLE sessions -( - session_id bigint PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - tracker_version text NOT NULL, - start_ts bigint NOT NULL, - duration integer NULL, - rev_id text DEFAULT NULL, - platform platform NOT NULL DEFAULT 'web', - is_snippet boolean NOT NULL DEFAULT FALSE, - user_id text DEFAULT NULL, - user_anonymous_id text DEFAULT NULL, - user_uuid uuid NOT NULL, - user_agent text DEFAULT NULL, - user_os text NOT NULL, - user_os_version text DEFAULT NULL, - user_browser text DEFAULT NULL, - user_browser_version text DEFAULT NULL, - user_device text NOT NULL, - user_device_type device_type NOT NULL, - user_device_memory_size integer DEFAULT NULL, - user_device_heap_size bigint DEFAULT NULL, - user_country country NOT NULL, - pages_count integer NOT NULL DEFAULT 0, - events_count integer NOT NULL DEFAULT 0, - errors_count integer NOT NULL DEFAULT 0, - watchdogs_score bigint NOT NULL DEFAULT 0, - issue_score bigint NOT NULL DEFAULT 0, - issue_types issue_type[] NOT NULL DEFAULT '{}'::issue_type[], - metadata_1 text DEFAULT NULL, - metadata_2 text DEFAULT NULL, - metadata_3 text DEFAULT NULL, - metadata_4 text DEFAULT NULL, - metadata_5 text DEFAULT NULL, - metadata_6 text DEFAULT NULL, - metadata_7 text DEFAULT NULL, - metadata_8 text DEFAULT NULL, - metadata_9 text DEFAULT NULL, - metadata_10 text DEFAULT NULL --- , --- rehydration_id integer REFERENCES rehydrations(rehydration_id) ON DELETE SET NULL -); -CREATE INDEX ON sessions (project_id, start_ts); -CREATE INDEX ON sessions (project_id, user_id); -CREATE INDEX ON sessions (project_id, user_anonymous_id); -CREATE INDEX ON sessions (project_id, user_device); -CREATE INDEX ON sessions (project_id, user_country); -CREATE INDEX ON sessions (project_id, user_browser); -CREATE INDEX ON sessions (project_id, metadata_1); -CREATE INDEX ON sessions (project_id, metadata_2); -CREATE INDEX ON sessions (project_id, metadata_3); -CREATE INDEX ON sessions (project_id, metadata_4); -CREATE INDEX ON sessions (project_id, metadata_5); -CREATE INDEX ON sessions (project_id, metadata_6); -CREATE INDEX ON sessions (project_id, metadata_7); -CREATE INDEX ON sessions (project_id, metadata_8); -CREATE INDEX ON sessions (project_id, metadata_9); -CREATE INDEX ON sessions (project_id, metadata_10); --- CREATE INDEX ON sessions (rehydration_id); -CREATE INDEX ON sessions (project_id, watchdogs_score DESC); -CREATE INDEX platform_idx ON public.sessions (platform); - -CREATE INDEX sessions_metadata1_gin_idx ON public.sessions USING GIN (metadata_1 gin_trgm_ops); -CREATE INDEX sessions_metadata2_gin_idx ON public.sessions USING GIN (metadata_2 gin_trgm_ops); -CREATE INDEX sessions_metadata3_gin_idx ON public.sessions USING GIN (metadata_3 gin_trgm_ops); -CREATE INDEX sessions_metadata4_gin_idx ON public.sessions USING GIN (metadata_4 gin_trgm_ops); -CREATE INDEX sessions_metadata5_gin_idx ON public.sessions USING GIN (metadata_5 gin_trgm_ops); -CREATE INDEX sessions_metadata6_gin_idx ON public.sessions USING GIN (metadata_6 gin_trgm_ops); -CREATE INDEX sessions_metadata7_gin_idx ON public.sessions USING GIN (metadata_7 gin_trgm_ops); -CREATE INDEX sessions_metadata8_gin_idx ON public.sessions USING GIN (metadata_8 gin_trgm_ops); -CREATE INDEX sessions_metadata9_gin_idx ON public.sessions USING GIN (metadata_9 gin_trgm_ops); -CREATE INDEX sessions_metadata10_gin_idx ON public.sessions USING GIN (metadata_10 gin_trgm_ops); -CREATE INDEX sessions_user_os_gin_idx ON public.sessions USING GIN (user_os gin_trgm_ops); -CREATE INDEX sessions_user_browser_gin_idx ON public.sessions USING GIN (user_browser gin_trgm_ops); -CREATE INDEX sessions_user_device_gin_idx ON public.sessions USING GIN (user_device gin_trgm_ops); -CREATE INDEX sessions_user_id_gin_idx ON public.sessions USING GIN (user_id gin_trgm_ops); -CREATE INDEX sessions_user_anonymous_id_gin_idx ON public.sessions USING GIN (user_anonymous_id gin_trgm_ops); -CREATE INDEX sessions_user_country_gin_idx ON public.sessions (project_id, user_country); -CREATE INDEX ON sessions (project_id, user_country); -CREATE INDEX ON sessions (project_id, user_browser); -CREATE INDEX sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; -CREATE INDEX sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; -CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; -CREATE INDEX sessions_session_id_project_id_start_ts_durationNN_idx ON sessions (session_id, project_id, start_ts) WHERE duration IS NOT NULL; -CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; -CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; - -ALTER TABLE public.sessions - ADD CONSTRAINT web_browser_constraint CHECK ( (sessions.platform = 'web' AND sessions.user_browser NOTNULL) OR - (sessions.platform != 'web' AND sessions.user_browser ISNULL)); - -ALTER TABLE public.sessions - ADD CONSTRAINT web_user_browser_version_constraint CHECK ( sessions.platform = 'web' OR sessions.user_browser_version ISNULL); - -ALTER TABLE public.sessions - ADD CONSTRAINT web_user_agent_constraint CHECK ( (sessions.platform = 'web' AND sessions.user_agent NOTNULL) OR - (sessions.platform != 'web' AND sessions.user_agent ISNULL)); - - - -CREATE TABLE user_viewed_sessions -( - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - PRIMARY KEY (user_id, session_id) -); - -CREATE TABLE user_favorite_sessions -( - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - PRIMARY KEY (user_id, session_id) -); - - --- --- assignments.sql --- - -create table assigned_sessions -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - issue_id text NOT NULL, - provider oauth_provider NOT NULL, - created_by integer NOT NULL, - created_at timestamp default timezone('utc'::text, now()) NOT NULL, - provider_data jsonb default '{}'::jsonb NOT NULL -); -CREATE INDEX ON assigned_sessions (session_id); - --- --- events_common.sql --- - -CREATE SCHEMA events_common; - -CREATE TYPE events_common.custom_level AS ENUM ('info','error'); - -CREATE TABLE events_common.customs -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - name text NOT NULL, - payload jsonb NOT NULL, - level events_common.custom_level NOT NULL DEFAULT 'info', - PRIMARY KEY (session_id, timestamp, seq_index) -); -CREATE INDEX ON events_common.customs (name); -CREATE INDEX customs_name_gin_idx ON events_common.customs USING GIN (name gin_trgm_ops); -CREATE INDEX ON events_common.customs (timestamp); - - -CREATE TABLE events_common.issues -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - issue_id text NOT NULL REFERENCES issues (issue_id) ON DELETE CASCADE, - payload jsonb DEFAULT NULL, - PRIMARY KEY (session_id, timestamp, seq_index) -); - - -CREATE TABLE events_common.requests -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - url text NOT NULL, - duration integer NOT NULL, - success boolean NOT NULL, - PRIMARY KEY (session_id, timestamp, seq_index) -); -CREATE INDEX ON events_common.requests (url); -CREATE INDEX ON events_common.requests (duration); -CREATE INDEX requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops); -CREATE INDEX ON events_common.requests (timestamp); -CREATE INDEX requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, length(url) - (CASE - WHEN url LIKE 'http://%' - THEN 7 - WHEN url LIKE 'https://%' - THEN 8 - ELSE 0 END)) - gin_trgm_ops); - -- --- events.sql --- -CREATE SCHEMA events; - -CREATE TABLE events.pages -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - host text NOT NULL, - path text NOT NULL, - base_path text NOT NULL, - referrer text DEFAULT NULL, - base_referrer text DEFAULT NULL, - dom_building_time integer DEFAULT NULL, - dom_content_loaded_time integer DEFAULT NULL, - load_time integer DEFAULT NULL, - first_paint_time integer DEFAULT NULL, - first_contentful_paint_time integer DEFAULT NULL, - speed_index integer DEFAULT NULL, - visually_complete integer DEFAULT NULL, - time_to_interactive integer DEFAULT NULL, - response_time bigint DEFAULT NULL, - response_end bigint DEFAULT NULL, - ttfb integer DEFAULT NULL, - PRIMARY KEY (session_id, message_id) -); -CREATE INDEX ON events.pages (session_id); -CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); -CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); -CREATE INDEX ON events.pages (timestamp); -CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); -CREATE INDEX pages_base_path_idx ON events.pages (base_path); -CREATE INDEX pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1)); -CREATE INDEX pages_base_referrer_idx ON events.pages (base_referrer); -CREATE INDEX pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer, length(base_referrer) - (CASE - WHEN base_referrer LIKE 'http://%' - THEN 7 - WHEN base_referrer LIKE 'https://%' - THEN 8 - ELSE 0 END)) - gin_trgm_ops); -CREATE INDEX ON events.pages (response_time); -CREATE INDEX ON events.pages (response_end); -CREATE INDEX pages_path_gin_idx ON events.pages USING GIN (path gin_trgm_ops); -CREATE INDEX pages_path_idx ON events.pages (path); -CREATE INDEX pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0; -CREATE INDEX pages_dom_building_time_idx ON events.pages (dom_building_time) WHERE dom_building_time > 0; -CREATE INDEX pages_load_time_idx ON events.pages (load_time) WHERE load_time > 0; -CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; -CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; -CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; -CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0; -CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0; -CREATE INDEX pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; -CREATE INDEX pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; -CREATE INDEX pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR first_paint_time > 0 OR - dom_content_loaded_time > 0 OR ttfb > 0 OR - time_to_interactive > 0; -CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; -CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; -CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); -CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; -CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); - -CREATE TABLE events.clicks -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - label text DEFAULT NULL, - url text DEFAULT '' NOT NULL, - selector text DEFAULT '' NOT NULL, - PRIMARY KEY (session_id, message_id) -); -CREATE INDEX ON events.clicks (session_id); -CREATE INDEX ON events.clicks (label); -CREATE INDEX clicks_label_gin_idx ON events.clicks USING GIN (label gin_trgm_ops); -CREATE INDEX ON events.clicks (timestamp); -CREATE INDEX clicks_label_session_id_timestamp_idx ON events.clicks (label, session_id, timestamp); -CREATE INDEX clicks_url_idx ON events.clicks (url); -CREATE INDEX clicks_url_gin_idx ON events.clicks USING GIN (url gin_trgm_ops); -CREATE INDEX clicks_url_session_id_timestamp_selector_idx ON events.clicks (url, session_id, timestamp, selector); -CREATE INDEX clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp); - - -CREATE TABLE events.inputs -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - label text DEFAULT NULL, - value text DEFAULT NULL, - PRIMARY KEY (session_id, message_id) -); -CREATE INDEX ON events.inputs (session_id); -CREATE INDEX ON events.inputs (label, value); -CREATE INDEX inputs_label_gin_idx ON events.inputs USING GIN (label gin_trgm_ops); -CREATE INDEX inputs_label_idx ON events.inputs (label); -CREATE INDEX ON events.inputs (timestamp); -CREATE INDEX inputs_label_session_id_timestamp_idx ON events.inputs (label, session_id, timestamp); - -CREATE TABLE events.errors -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, - PRIMARY KEY (session_id, message_id) -); -CREATE INDEX ON events.errors (session_id); -CREATE INDEX errors_error_id_idx ON events.errors (error_id); -CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); -CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); -CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); -CREATE INDEX errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); - -CREATE TABLE events.graphql -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - name text NOT NULL, - PRIMARY KEY (session_id, message_id) -); -CREATE INDEX ON events.graphql (name); -CREATE INDEX graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); -CREATE INDEX ON events.graphql (timestamp); - -CREATE TABLE events.state_actions -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - name text NOT NULL, - PRIMARY KEY (session_id, message_id) -); -CREATE INDEX ON events.state_actions (name); -CREATE INDEX state_actions_name_gin_idx ON events.state_actions USING GIN (name gin_trgm_ops); -CREATE INDEX ON events.state_actions (timestamp); - -CREATE TYPE events.resource_type AS ENUM ('other', 'script', 'stylesheet', 'fetch', 'img', 'media'); -CREATE TYPE events.resource_method AS ENUM ('GET' , 'HEAD' , 'POST' , 'PUT' , 'DELETE' , 'CONNECT' , 'OPTIONS' , 'TRACE' , 'PATCH' ); -CREATE TABLE events.resources -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - duration bigint NULL, - type events.resource_type NOT NULL, - url text NOT NULL, - url_host text NOT NULL, - url_hostpath text NOT NULL, - success boolean NOT NULL, - status smallint NULL, - method events.resource_method NULL, - ttfb bigint NULL, - header_size bigint NULL, - encoded_body_size integer NULL, - decoded_body_size integer NULL, - PRIMARY KEY (session_id, message_id) -); -CREATE INDEX ON events.resources (session_id); -CREATE INDEX ON events.resources (status); -CREATE INDEX ON events.resources (type); -CREATE INDEX ON events.resources (duration) WHERE duration > 0; -CREATE INDEX ON events.resources (url_host); - -CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_ops); -CREATE INDEX resources_url_idx ON events.resources (url); -CREATE INDEX resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops); -CREATE INDEX resources_url_hostpath_idx ON events.resources (url_hostpath); -CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; -CREATE INDEX resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); -CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); -CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; -CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; -CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); -CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; -CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); - -CREATE TABLE events.performance -( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - message_id bigint NOT NULL, - min_fps smallint NOT NULL, - avg_fps smallint NOT NULL, - max_fps smallint NOT NULL, - min_cpu smallint NOT NULL, - avg_cpu smallint NOT NULL, - max_cpu smallint NOT NULL, - min_total_js_heap_size bigint NOT NULL, - avg_total_js_heap_size bigint NOT NULL, - max_total_js_heap_size bigint NOT NULL, - min_used_js_heap_size bigint NOT NULL, - avg_used_js_heap_size bigint NOT NULL, - max_used_js_heap_size bigint NOT NULL, - PRIMARY KEY (session_id, message_id) -); - CREATE OR REPLACE FUNCTION events.funnel(steps integer[], m integer) RETURNS boolean AS $$ @@ -843,39 +48,866 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +-- --- integrations.sql --- + +CREATE OR REPLACE FUNCTION notify_integration() RETURNS trigger AS +$$ +BEGIN + IF NEW IS NULL THEN + PERFORM pg_notify('integration', (row_to_json(OLD)::text || '{"options": null, "request_data": null}'::text)); + ELSIF (OLD IS NULL) OR (OLD.options <> NEW.options) THEN + PERFORM pg_notify('integration', row_to_json(NEW)::text); + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +-- --- alerts.sql --- + +CREATE OR REPLACE FUNCTION notify_alert() RETURNS trigger AS +$$ +DECLARE + clone jsonb; +BEGIN + clone = to_jsonb(NEW); + clone = jsonb_set(clone, '{created_at}', to_jsonb(CAST(EXTRACT(epoch FROM NEW.created_at) * 1000 AS BIGINT))); + IF NEW.deleted_at NOTNULL THEN + clone = jsonb_set(clone, '{deleted_at}', to_jsonb(CAST(EXTRACT(epoch FROM NEW.deleted_at) * 1000 AS BIGINT))); + END IF; + PERFORM pg_notify('alert', clone::text); + RETURN NEW; +END ; +$$ LANGUAGE plpgsql; + +-- --- projects.sql --- + +CREATE OR REPLACE FUNCTION notify_project() RETURNS trigger AS +$$ +BEGIN + PERFORM pg_notify('project', row_to_json(NEW)::text); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- All tables and types: + +DO +$$ + BEGIN + IF EXISTS(SELECT + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'tenants') THEN + raise notice 'DB exists, skipping creation query'; + ELSE + raise notice 'Creating DB'; + + -- --- public.sql --- + + CREATE EXTENSION IF NOT EXISTS pg_trgm; + CREATE EXTENSION IF NOT EXISTS pgcrypto; +-- --- accounts.sql --- + + CREATE TABLE IF NOT EXISTS public.tenants + ( + tenant_id integer NOT NULL DEFAULT 1, + user_id text NOT NULL DEFAULT generate_api_key(20), + name text NOT NULL, + api_key text NOT NULL DEFAULT generate_api_key(20), + created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), + edition varchar(3) NOT NULL, + version_number text NOT NULL, + license text NULL, + opt_out bool NOT NULL DEFAULT FALSE, + t_projects integer NOT NULL DEFAULT 1, + t_sessions bigint NOT NULL DEFAULT 0, + t_users integer NOT NULL DEFAULT 1, + t_integrations integer NOT NULL DEFAULT 0, + CONSTRAINT onerow_uni CHECK (tenant_id = 1) + ); + + CREATE TYPE user_role AS ENUM ('owner', 'admin', 'member'); + + CREATE TABLE users + ( + user_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + email text NOT NULL UNIQUE, + role user_role NOT NULL DEFAULT 'member', + name text NOT NULL, + created_at timestamp without time zone NOT NULL default (now() at time zone 'utc'), + deleted_at timestamp without time zone NULL DEFAULT NULL, + appearance jsonb NOT NULL default '{ + "role": "dev", + "dashboard": { + "cpu": true, + "fps": false, + "avgCpu": true, + "avgFps": true, + "errors": true, + "crashes": true, + "overview": true, + "sessions": true, + "topMetrics": true, + "callsErrors": true, + "pageMetrics": true, + "performance": true, + "timeToRender": false, + "userActivity": false, + "avgFirstPaint": false, + "countSessions": true, + "errorsPerType": true, + "slowestImages": true, + "speedLocation": true, + "slowestDomains": true, + "avgPageLoadTime": true, + "avgTillFirstBit": false, + "avgTimeToRender": true, + "avgVisitedPages": false, + "avgImageLoadTime": true, + "busiestTimeOfDay": true, + "errorsPerDomains": true, + "missingResources": true, + "resourcesByParty": true, + "sessionsFeedback": false, + "slowestResources": true, + "avgUsedJsHeapSize": true, + "domainsErrors_4xx": true, + "domainsErrors_5xx": true, + "memoryConsumption": true, + "pagesDomBuildtime": false, + "pagesResponseTime": true, + "avgRequestLoadTime": true, + "avgSessionDuration": false, + "sessionsPerBrowser": false, + "applicationActivity": true, + "sessionsFrustration": false, + "avgPagesDomBuildtime": true, + "avgPagesResponseTime": false, + "avgTimeToInteractive": true, + "resourcesCountByType": true, + "resourcesLoadingTime": true, + "avgDomContentLoadStart": true, + "avgFirstContentfulPixel": false, + "resourceTypeVsResponseEnd": true, + "impactedSessionsByJsErrors": true, + "impactedSessionsBySlowPages": true, + "resourcesVsVisuallyComplete": true, + "pagesResponseTimeDistribution": true + }, + "sessionsLive": false, + "sessionsDevtools": true + }'::jsonb, + api_key text UNIQUE default generate_api_key(20) not null, + jwt_iat timestamp without time zone NULL DEFAULT NULL, + data jsonb NOT NULL DEFAULT '{}'::jsonb, + weekly_report boolean NOT NULL DEFAULT TRUE + ); + + CREATE TABLE basic_authentication + ( + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + password text DEFAULT NULL, + generated_password boolean NOT NULL DEFAULT false, + invitation_token text NULL DEFAULT NULL, + invited_at timestamp without time zone NULL DEFAULT NULL, + change_pwd_token text NULL DEFAULT NULL, + change_pwd_expire_at timestamp without time zone NULL DEFAULT NULL, + changed_at timestamp, + UNIQUE (user_id) + ); + + CREATE TYPE oauth_provider AS ENUM ('jira', 'github'); + CREATE TABLE oauth_authentication + ( + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + provider oauth_provider NOT NULL, + provider_user_id text NOT NULL, + token text NOT NULL, + UNIQUE (user_id, provider) + ); + +-- --- projects.sql --- + + CREATE TABLE projects + ( + project_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_key varchar(20) NOT NULL UNIQUE DEFAULT generate_api_key(20), + name text NOT NULL, + active boolean NOT NULL, + sample_rate smallint NOT NULL DEFAULT 100 CHECK (sample_rate >= 0 AND sample_rate <= 100), + created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), + deleted_at timestamp without time zone NULL DEFAULT NULL, + max_session_duration integer NOT NULL DEFAULT 7200000, + metadata_1 text DEFAULT NULL, + metadata_2 text DEFAULT NULL, + metadata_3 text DEFAULT NULL, + metadata_4 text DEFAULT NULL, + metadata_5 text DEFAULT NULL, + metadata_6 text DEFAULT NULL, + metadata_7 text DEFAULT NULL, + metadata_8 text DEFAULT NULL, + metadata_9 text DEFAULT NULL, + metadata_10 text DEFAULT NULL, + gdpr jsonb NOT NULL DEFAULT '{ + "maskEmails": true, + "sampleRate": 33, + "maskNumbers": false, + "defaultInputMode": "plain" + }'::jsonb -- ?????? + ); + + CREATE TRIGGER on_insert_or_update + AFTER INSERT OR UPDATE + ON projects + FOR EACH ROW + EXECUTE PROCEDURE notify_project(); + +-- --- alerts.sql --- + + CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); + + CREATE TABLE alerts + ( + alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + name text NOT NULL, + description text NULL DEFAULT NULL, + active boolean NOT NULL DEFAULT TRUE, + detection_method alert_detection_method NOT NULL, + query jsonb NOT NULL, + deleted_at timestamp NULL DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + options jsonb NOT NULL DEFAULT '{ + "renotifyInterval": 1440 + }'::jsonb + ); + + + CREATE TRIGGER on_insert_or_update_or_delete + AFTER INSERT OR UPDATE OR DELETE + ON alerts + FOR EACH ROW + EXECUTE PROCEDURE notify_alert(); + +-- --- webhooks.sql --- + + create type webhook_type as enum ('webhook', 'slack', 'email'); + + create table webhooks + ( + webhook_id integer generated by default as identity + constraint webhooks_pkey + primary key, + endpoint text not null, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + auth_header text, + type webhook_type not null, + index integer default 0 not null, + name varchar(100) + ); + +-- --- notifications.sql --- + + CREATE TABLE notifications + ( + notification_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer REFERENCES users (user_id) ON DELETE CASCADE, + title text NOT NULL, + description text NOT NULL, + button_text varchar(80) NULL, + button_url text NULL, + image_url text NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + options jsonb NOT NULL DEFAULT '{}'::jsonb + ); + + CREATE INDEX notifications_user_id_index ON notifications (user_id); + CREATE INDEX notifications_created_at_index ON notifications (created_at DESC); + CREATE INDEX notifications_created_at_epoch_idx ON notifications (CAST(EXTRACT(EPOCH FROM created_at) * 1000 AS BIGINT) DESC); + + CREATE TABLE user_viewed_notifications + ( + user_id integer NOT NULL REFERENCES users (user_id) on delete cascade, + notification_id integer NOT NULL REFERENCES notifications (notification_id) on delete cascade, + constraint user_viewed_notifications_pkey primary key (user_id, notification_id) + ); + +-- --- funnels.sql --- + + CREATE TABLE funnels + ( + funnel_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + name text not null, + filter jsonb not null, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + is_public boolean NOT NULL DEFAULT False + ); + + CREATE INDEX ON public.funnels (user_id, is_public); + +-- --- announcements.sql --- + + create type announcement_type as enum ('notification', 'alert'); + + create table announcements + ( + announcement_id serial not null + constraint announcements_pk + primary key, + title text not null, + description text not null, + button_text varchar(30), + button_url text, + image_url text, + created_at timestamp default timezone('utc'::text, now()) not null, + type announcement_type default 'notification'::announcement_type not null + ); + +-- --- integrations.sql --- + + CREATE TYPE integration_provider AS ENUM ('bugsnag', 'cloudwatch', 'datadog', 'newrelic', 'rollbar', 'sentry', 'stackdriver', 'sumologic', 'elasticsearch'); --, 'jira', 'github'); + CREATE TABLE integrations + ( + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + provider integration_provider NOT NULL, + options jsonb NOT NULL, + request_data jsonb NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (project_id, provider) + ); + + CREATE TRIGGER on_insert_or_update_or_delete + AFTER INSERT OR UPDATE OR DELETE + ON integrations + FOR EACH ROW + EXECUTE PROCEDURE notify_integration(); + + + create table jira_cloud + ( + user_id integer not null + constraint jira_cloud_pk + primary key + constraint jira_cloud_users_fkey + references users + on delete cascade, + username text not null, + token text not null, + url text + ); + +-- --- issues.sql --- + + CREATE TYPE issue_type AS ENUM ( + 'click_rage', + 'dead_click', + 'excessive_scrolling', + 'bad_request', + 'missing_resource', + 'memory', + 'cpu', + 'slow_resource', + 'slow_page_load', + 'crash', + 'ml_cpu', + 'ml_memory', + 'ml_dead_click', + 'ml_click_rage', + 'ml_mouse_thrashing', + 'ml_excessive_scrolling', + 'ml_slow_resources', + 'custom', + 'js_exception' + ); + + CREATE TABLE issues + ( + issue_id text NOT NULL PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + type issue_type NOT NULL, + context_string text NOT NULL, + context jsonb DEFAULT NULL + ); + CREATE INDEX ON issues (issue_id, type); + CREATE INDEX issues_context_string_gin_idx ON public.issues USING GIN (context_string gin_trgm_ops); + CREATE INDEX issues_project_id_idx ON issues (project_id); + +-- --- errors.sql --- + + CREATE TYPE error_source AS ENUM ('js_exception', 'bugsnag', 'cloudwatch', 'datadog', 'newrelic', 'rollbar', 'sentry', 'stackdriver', 'sumologic'); + CREATE TYPE error_status AS ENUM ('unresolved', 'resolved', 'ignored'); + CREATE TABLE errors + ( + error_id text NOT NULL PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + source error_source NOT NULL, + name text DEFAULT NULL, + message text NOT NULL, + payload jsonb NOT NULL, + status error_status NOT NULL DEFAULT 'unresolved', + parent_error_id text DEFAULT NULL REFERENCES errors (error_id) ON DELETE SET NULL, + stacktrace jsonb, --to save the stacktrace and not query S3 another time + stacktrace_parsed_at timestamp + ); + CREATE INDEX ON errors (project_id, source); + CREATE INDEX errors_message_gin_idx ON public.errors USING GIN (message gin_trgm_ops); + CREATE INDEX errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops); + CREATE INDEX errors_project_id_idx ON public.errors (project_id); + CREATE INDEX errors_project_id_status_idx ON public.errors (project_id, status); + CREATE INDEX errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception'; + CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id); + CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; + + CREATE TABLE user_favorite_errors + ( + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, + PRIMARY KEY (user_id, error_id) + ); + + CREATE TABLE user_viewed_errors + ( + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, + PRIMARY KEY (user_id, error_id) + ); + CREATE INDEX user_viewed_errors_user_id_idx ON public.user_viewed_errors (user_id); + CREATE INDEX user_viewed_errors_error_id_idx ON public.user_viewed_errors (error_id); + + +-- --- sessions.sql --- + CREATE TYPE device_type AS ENUM ('desktop', 'tablet', 'mobile', 'other'); + CREATE TYPE country AS ENUM ('UN', 'RW', 'SO', 'YE', 'IQ', 'SA', 'IR', 'CY', 'TZ', 'SY', 'AM', 'KE', 'CD', 'DJ', 'UG', 'CF', 'SC', 'JO', 'LB', 'KW', 'OM', 'QA', 'BH', 'AE', 'IL', 'TR', 'ET', 'ER', 'EG', 'SD', 'GR', 'BI', 'EE', 'LV', 'AZ', 'LT', 'SJ', 'GE', 'MD', 'BY', 'FI', 'AX', 'UA', 'MK', 'HU', 'BG', 'AL', 'PL', 'RO', 'XK', 'ZW', 'ZM', 'KM', 'MW', 'LS', 'BW', 'MU', 'SZ', 'RE', 'ZA', 'YT', 'MZ', 'MG', 'AF', 'PK', 'BD', 'TM', 'TJ', 'LK', 'BT', 'IN', 'MV', 'IO', 'NP', 'MM', 'UZ', 'KZ', 'KG', 'TF', 'HM', 'CC', 'PW', 'VN', 'TH', 'ID', 'LA', 'TW', 'PH', 'MY', 'CN', 'HK', 'BN', 'MO', 'KH', 'KR', 'JP', 'KP', 'SG', 'CK', 'TL', 'RU', 'MN', 'AU', 'CX', 'MH', 'FM', 'PG', 'SB', 'TV', 'NR', 'VU', 'NC', 'NF', 'NZ', 'FJ', 'LY', 'CM', 'SN', 'CG', 'PT', 'LR', 'CI', 'GH', 'GQ', 'NG', 'BF', 'TG', 'GW', 'MR', 'BJ', 'GA', 'SL', 'ST', 'GI', 'GM', 'GN', 'TD', 'NE', 'ML', 'EH', 'TN', 'ES', 'MA', 'MT', 'DZ', 'FO', 'DK', 'IS', 'GB', 'CH', 'SE', 'NL', 'AT', 'BE', 'DE', 'LU', 'IE', 'MC', 'FR', 'AD', 'LI', 'JE', 'IM', 'GG', 'SK', 'CZ', 'NO', 'VA', 'SM', 'IT', 'SI', 'ME', 'HR', 'BA', 'AO', 'NA', 'SH', 'BV', 'BB', 'CV', 'GY', 'GF', 'SR', 'PM', 'GL', 'PY', 'UY', 'BR', 'FK', 'GS', 'JM', 'DO', 'CU', 'MQ', 'BS', 'BM', 'AI', 'TT', 'KN', 'DM', 'AG', 'LC', 'TC', 'AW', 'VG', 'VC', 'MS', 'MF', 'BL', 'GP', 'GD', 'KY', 'BZ', 'SV', 'GT', 'HN', 'NI', 'CR', 'VE', 'EC', 'CO', 'PA', 'HT', 'AR', 'CL', 'BO', 'PE', 'MX', 'PF', 'PN', 'KI', 'TK', 'TO', 'WF', 'WS', 'NU', 'MP', 'GU', 'PR', 'VI', 'UM', 'AS', 'CA', 'US', 'PS', 'RS', 'AQ', 'SX', 'CW', 'BQ', 'SS'); + CREATE TYPE platform AS ENUM ('web','ios','android'); + + CREATE TABLE sessions + ( + session_id bigint PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + tracker_version text NOT NULL, + start_ts bigint NOT NULL, + duration integer NULL, + rev_id text DEFAULT NULL, + platform platform NOT NULL DEFAULT 'web', + is_snippet boolean NOT NULL DEFAULT FALSE, + user_id text DEFAULT NULL, + user_anonymous_id text DEFAULT NULL, + user_uuid uuid NOT NULL, + user_agent text DEFAULT NULL, + user_os text NOT NULL, + user_os_version text DEFAULT NULL, + user_browser text DEFAULT NULL, + user_browser_version text DEFAULT NULL, + user_device text NOT NULL, + user_device_type device_type NOT NULL, + user_device_memory_size integer DEFAULT NULL, + user_device_heap_size bigint DEFAULT NULL, + user_country country NOT NULL, + pages_count integer NOT NULL DEFAULT 0, + events_count integer NOT NULL DEFAULT 0, + errors_count integer NOT NULL DEFAULT 0, + watchdogs_score bigint NOT NULL DEFAULT 0, + issue_score bigint NOT NULL DEFAULT 0, + issue_types issue_type[] NOT NULL DEFAULT '{}'::issue_type[], + metadata_1 text DEFAULT NULL, + metadata_2 text DEFAULT NULL, + metadata_3 text DEFAULT NULL, + metadata_4 text DEFAULT NULL, + metadata_5 text DEFAULT NULL, + metadata_6 text DEFAULT NULL, + metadata_7 text DEFAULT NULL, + metadata_8 text DEFAULT NULL, + metadata_9 text DEFAULT NULL, + metadata_10 text DEFAULT NULL +-- , +-- rehydration_id integer REFERENCES rehydrations(rehydration_id) ON DELETE SET NULL + ); + CREATE INDEX ON sessions (project_id, start_ts); + CREATE INDEX ON sessions (project_id, user_id); + CREATE INDEX ON sessions (project_id, user_anonymous_id); + CREATE INDEX ON sessions (project_id, user_device); + CREATE INDEX ON sessions (project_id, user_country); + CREATE INDEX ON sessions (project_id, user_browser); + CREATE INDEX ON sessions (project_id, metadata_1); + CREATE INDEX ON sessions (project_id, metadata_2); + CREATE INDEX ON sessions (project_id, metadata_3); + CREATE INDEX ON sessions (project_id, metadata_4); + CREATE INDEX ON sessions (project_id, metadata_5); + CREATE INDEX ON sessions (project_id, metadata_6); + CREATE INDEX ON sessions (project_id, metadata_7); + CREATE INDEX ON sessions (project_id, metadata_8); + CREATE INDEX ON sessions (project_id, metadata_9); + CREATE INDEX ON sessions (project_id, metadata_10); +-- CREATE INDEX ON sessions (rehydration_id); + CREATE INDEX ON sessions (project_id, watchdogs_score DESC); + CREATE INDEX platform_idx ON public.sessions (platform); + + CREATE INDEX sessions_metadata1_gin_idx ON public.sessions USING GIN (metadata_1 gin_trgm_ops); + CREATE INDEX sessions_metadata2_gin_idx ON public.sessions USING GIN (metadata_2 gin_trgm_ops); + CREATE INDEX sessions_metadata3_gin_idx ON public.sessions USING GIN (metadata_3 gin_trgm_ops); + CREATE INDEX sessions_metadata4_gin_idx ON public.sessions USING GIN (metadata_4 gin_trgm_ops); + CREATE INDEX sessions_metadata5_gin_idx ON public.sessions USING GIN (metadata_5 gin_trgm_ops); + CREATE INDEX sessions_metadata6_gin_idx ON public.sessions USING GIN (metadata_6 gin_trgm_ops); + CREATE INDEX sessions_metadata7_gin_idx ON public.sessions USING GIN (metadata_7 gin_trgm_ops); + CREATE INDEX sessions_metadata8_gin_idx ON public.sessions USING GIN (metadata_8 gin_trgm_ops); + CREATE INDEX sessions_metadata9_gin_idx ON public.sessions USING GIN (metadata_9 gin_trgm_ops); + CREATE INDEX sessions_metadata10_gin_idx ON public.sessions USING GIN (metadata_10 gin_trgm_ops); + CREATE INDEX sessions_user_os_gin_idx ON public.sessions USING GIN (user_os gin_trgm_ops); + CREATE INDEX sessions_user_browser_gin_idx ON public.sessions USING GIN (user_browser gin_trgm_ops); + CREATE INDEX sessions_user_device_gin_idx ON public.sessions USING GIN (user_device gin_trgm_ops); + CREATE INDEX sessions_user_id_gin_idx ON public.sessions USING GIN (user_id gin_trgm_ops); + CREATE INDEX sessions_user_anonymous_id_gin_idx ON public.sessions USING GIN (user_anonymous_id gin_trgm_ops); + CREATE INDEX sessions_user_country_gin_idx ON public.sessions (project_id, user_country); + CREATE INDEX ON sessions (project_id, user_country); + CREATE INDEX ON sessions (project_id, user_browser); + CREATE INDEX sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; + CREATE INDEX sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; + CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; + CREATE INDEX sessions_session_id_project_id_start_ts_durationNN_idx ON sessions (session_id, project_id, start_ts) WHERE duration IS NOT NULL; + CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; + CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; + + ALTER TABLE public.sessions + ADD CONSTRAINT web_browser_constraint CHECK ( + (sessions.platform = 'web' AND sessions.user_browser NOTNULL) OR + (sessions.platform != 'web' AND sessions.user_browser ISNULL)); + + ALTER TABLE public.sessions + ADD CONSTRAINT web_user_browser_version_constraint CHECK ( sessions.platform = 'web' OR sessions.user_browser_version ISNULL); + + ALTER TABLE public.sessions + ADD CONSTRAINT web_user_agent_constraint CHECK ( + (sessions.platform = 'web' AND sessions.user_agent NOTNULL) OR + (sessions.platform != 'web' AND sessions.user_agent ISNULL)); + + + CREATE TABLE user_viewed_sessions + ( + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + PRIMARY KEY (user_id, session_id) + ); + + CREATE TABLE user_favorite_sessions + ( + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + PRIMARY KEY (user_id, session_id) + ); + + +-- --- assignments.sql --- + + create table assigned_sessions + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + issue_id text NOT NULL, + provider oauth_provider NOT NULL, + created_by integer NOT NULL, + created_at timestamp default timezone('utc'::text, now()) NOT NULL, + provider_data jsonb default '{}'::jsonb NOT NULL + ); + CREATE INDEX ON assigned_sessions (session_id); + +-- --- events_common.sql --- + + CREATE SCHEMA IF NOT EXISTS events_common; + + CREATE TYPE events_common.custom_level AS ENUM ('info','error'); + + CREATE TABLE events_common.customs + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + name text NOT NULL, + payload jsonb NOT NULL, + level events_common.custom_level NOT NULL DEFAULT 'info', + PRIMARY KEY (session_id, timestamp, seq_index) + ); + CREATE INDEX ON events_common.customs (name); + CREATE INDEX customs_name_gin_idx ON events_common.customs USING GIN (name gin_trgm_ops); + CREATE INDEX ON events_common.customs (timestamp); + + + CREATE TABLE events_common.issues + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + issue_id text NOT NULL REFERENCES issues (issue_id) ON DELETE CASCADE, + payload jsonb DEFAULT NULL, + PRIMARY KEY (session_id, timestamp, seq_index) + ); + + + CREATE TABLE events_common.requests + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + url text NOT NULL, + duration integer NOT NULL, + success boolean NOT NULL, + PRIMARY KEY (session_id, timestamp, seq_index) + ); + CREATE INDEX ON events_common.requests (url); + CREATE INDEX ON events_common.requests (duration); + CREATE INDEX requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops); + CREATE INDEX ON events_common.requests (timestamp); + CREATE INDEX requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, length(url) - (CASE + WHEN url LIKE 'http://%' + THEN 7 + WHEN url LIKE 'https://%' + THEN 8 + ELSE 0 END)) + gin_trgm_ops); + +-- --- events.sql --- + CREATE SCHEMA IF NOT EXISTS events; + + CREATE TABLE events.pages + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + host text NOT NULL, + path text NOT NULL, + base_path text NOT NULL, + referrer text DEFAULT NULL, + base_referrer text DEFAULT NULL, + dom_building_time integer DEFAULT NULL, + dom_content_loaded_time integer DEFAULT NULL, + load_time integer DEFAULT NULL, + first_paint_time integer DEFAULT NULL, + first_contentful_paint_time integer DEFAULT NULL, + speed_index integer DEFAULT NULL, + visually_complete integer DEFAULT NULL, + time_to_interactive integer DEFAULT NULL, + response_time bigint DEFAULT NULL, + response_end bigint DEFAULT NULL, + ttfb integer DEFAULT NULL, + PRIMARY KEY (session_id, message_id) + ); + CREATE INDEX ON events.pages (session_id); + CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); + CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); + CREATE INDEX ON events.pages (timestamp); + CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); + CREATE INDEX pages_base_path_idx ON events.pages (base_path); + CREATE INDEX pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1)); + CREATE INDEX pages_base_referrer_idx ON events.pages (base_referrer); + CREATE INDEX pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer, + length(base_referrer) - (CASE + WHEN base_referrer LIKE 'http://%' + THEN 7 + WHEN base_referrer LIKE 'https://%' + THEN 8 + ELSE 0 END)) + gin_trgm_ops); + CREATE INDEX ON events.pages (response_time); + CREATE INDEX ON events.pages (response_end); + CREATE INDEX pages_path_gin_idx ON events.pages USING GIN (path gin_trgm_ops); + CREATE INDEX pages_path_idx ON events.pages (path); + CREATE INDEX pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0; + CREATE INDEX pages_dom_building_time_idx ON events.pages (dom_building_time) WHERE dom_building_time > 0; + CREATE INDEX pages_load_time_idx ON events.pages (load_time) WHERE load_time > 0; + CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; + CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; + CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; + CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0; + CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0; + CREATE INDEX pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; + CREATE INDEX pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; + CREATE INDEX pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR + first_paint_time > 0 OR + dom_content_loaded_time > 0 OR + ttfb > 0 OR + time_to_interactive > 0; + CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; + CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; + CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); + CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; + + + CREATE TABLE events.clicks + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + label text DEFAULT NULL, + url text DEFAULT '' NOT NULL, + selector text DEFAULT '' NOT NULL, + PRIMARY KEY (session_id, message_id) + ); + CREATE INDEX ON events.clicks (session_id); + CREATE INDEX ON events.clicks (label); + CREATE INDEX clicks_label_gin_idx ON events.clicks USING GIN (label gin_trgm_ops); + CREATE INDEX ON events.clicks (timestamp); + CREATE INDEX clicks_label_session_id_timestamp_idx ON events.clicks (label, session_id, timestamp); + CREATE INDEX clicks_url_idx ON events.clicks (url); + CREATE INDEX clicks_url_gin_idx ON events.clicks USING GIN (url gin_trgm_ops); + CREATE INDEX clicks_url_session_id_timestamp_selector_idx ON events.clicks (url, session_id, timestamp, selector); + + + CREATE TABLE events.inputs + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + label text DEFAULT NULL, + value text DEFAULT NULL, + PRIMARY KEY (session_id, message_id) + ); + CREATE INDEX ON events.inputs (session_id); + CREATE INDEX ON events.inputs (label, value); + CREATE INDEX inputs_label_gin_idx ON events.inputs USING GIN (label gin_trgm_ops); + CREATE INDEX inputs_label_idx ON events.inputs (label); + CREATE INDEX ON events.inputs (timestamp); + CREATE INDEX inputs_label_session_id_timestamp_idx ON events.inputs (label, session_id, timestamp); + + CREATE TABLE events.errors + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, + PRIMARY KEY (session_id, message_id) + ); + CREATE INDEX ON events.errors (session_id); + CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); + CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); + CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); + CREATE INDEX errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); + + CREATE TABLE events.graphql + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + name text NOT NULL, + PRIMARY KEY (session_id, message_id) + ); + CREATE INDEX ON events.graphql (name); + CREATE INDEX graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); + CREATE INDEX ON events.graphql (timestamp); + + CREATE TABLE events.state_actions + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + name text NOT NULL, + PRIMARY KEY (session_id, message_id) + ); + CREATE INDEX ON events.state_actions (name); + CREATE INDEX state_actions_name_gin_idx ON events.state_actions USING GIN (name gin_trgm_ops); + CREATE INDEX ON events.state_actions (timestamp); + + CREATE TYPE events.resource_type AS ENUM ('other', 'script', 'stylesheet', 'fetch', 'img', 'media'); + CREATE TYPE events.resource_method AS ENUM ('GET' , 'HEAD' , 'POST' , 'PUT' , 'DELETE' , 'CONNECT' , 'OPTIONS' , 'TRACE' , 'PATCH' ); + CREATE TABLE events.resources + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + duration bigint NULL, + type events.resource_type NOT NULL, + url text NOT NULL, + url_host text NOT NULL, + url_hostpath text NOT NULL, + success boolean NOT NULL, + status smallint NULL, + method events.resource_method NULL, + ttfb bigint NULL, + header_size bigint NULL, + encoded_body_size integer NULL, + decoded_body_size integer NULL, + PRIMARY KEY (session_id, message_id) + ); + CREATE INDEX ON events.resources (session_id); + CREATE INDEX ON events.resources (status); + CREATE INDEX ON events.resources (type); + CREATE INDEX ON events.resources (duration) WHERE duration > 0; + CREATE INDEX ON events.resources (url_host); + + CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_ops); + CREATE INDEX resources_url_idx ON events.resources (url); + CREATE INDEX resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops); + CREATE INDEX resources_url_hostpath_idx ON events.resources (url_hostpath); + CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; + CREATE INDEX resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); + CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); + CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; + CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; + CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); + CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; + CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); + + CREATE TABLE events.performance + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + message_id bigint NOT NULL, + min_fps smallint NOT NULL, + avg_fps smallint NOT NULL, + max_fps smallint NOT NULL, + min_cpu smallint NOT NULL, + avg_cpu smallint NOT NULL, + max_cpu smallint NOT NULL, + min_total_js_heap_size bigint NOT NULL, + avg_total_js_heap_size bigint NOT NULL, + max_total_js_heap_size bigint NOT NULL, + min_used_js_heap_size bigint NOT NULL, + avg_used_js_heap_size bigint NOT NULL, + max_used_js_heap_size bigint NOT NULL, + PRIMARY KEY (session_id, message_id) + ); + -- --- autocomplete.sql --- -CREATE TABLE autocomplete -( - value text NOT NULL, - type text NOT NULL, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE -); + CREATE TABLE autocomplete + ( + value text NOT NULL, + type text NOT NULL, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE + ); -CREATE unique index autocomplete_unique ON autocomplete (project_id, value, type); -CREATE index autocomplete_project_id_idx ON autocomplete (project_id); -CREATE INDEX autocomplete_type_idx ON public.autocomplete (type); -CREATE INDEX autocomplete_value_gin_idx ON public.autocomplete USING GIN (value gin_trgm_ops); + CREATE unique index autocomplete_unique ON autocomplete (project_id, value, type); + CREATE index autocomplete_project_id_idx ON autocomplete (project_id); + CREATE INDEX autocomplete_type_idx ON public.autocomplete (type); + CREATE INDEX autocomplete_value_gin_idx ON public.autocomplete USING GIN (value gin_trgm_ops); -- --- jobs.sql --- -CREATE TYPE job_status AS ENUM ('scheduled','running','cancelled','failed','completed'); -CREATE TYPE job_action AS ENUM ('delete_user_data'); -CREATE TABLE jobs -( - job_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - description text NOT NULL, - status job_status NOT NULL, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - action job_action NOT NULL, - reference_id text NOT NULL, - created_at timestamp default timezone('utc'::text, now()) NOT NULL, - updated_at timestamp default timezone('utc'::text, now()) NULL, - start_at timestamp NOT NULL, - errors text NULL -); -CREATE INDEX ON jobs (status); -CREATE INDEX ON jobs (start_at); -CREATE INDEX jobs_project_id_idx ON jobs (project_id); + CREATE TYPE job_status AS ENUM ('scheduled','running','cancelled','failed','completed'); + CREATE TYPE job_action AS ENUM ('delete_user_data'); + CREATE TABLE jobs + ( + job_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + description text NOT NULL, + status job_status NOT NULL, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + action job_action NOT NULL, + reference_id text NOT NULL, + created_at timestamp default timezone('utc'::text, now()) NOT NULL, + updated_at timestamp default timezone('utc'::text, now()) NULL, + start_at timestamp NOT NULL, + errors text NULL + ); + CREATE INDEX ON jobs (status); + CREATE INDEX ON jobs (start_at); + CREATE INDEX jobs_project_id_idx ON jobs (project_id); -COMMIT; + + raise notice 'DB created'; + END IF; + END; + +$$ +LANGUAGE plpgsql; + +COMMIT; \ No newline at end of file diff --git a/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml b/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml index 84cc6337d..2e6f259a1 100644 --- a/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml +++ b/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml @@ -5,6 +5,9 @@ metadata: namespace: {{ .Release.Namespace }} data: location.list: |- + location ~* /general_stats { + deny all; + } location /healthz { return 200 'OK'; } diff --git a/scripts/helm/roles/openreplay/tasks/install-dbs.yaml b/scripts/helm/roles/openreplay/tasks/install-dbs.yaml index 443e7a14f..ef7c1ff2f 100644 --- a/scripts/helm/roles/openreplay/tasks/install-dbs.yaml +++ b/scripts/helm/roles/openreplay/tasks/install-dbs.yaml @@ -21,7 +21,7 @@ file="{{ item|basename }}" kubectl exec -n db postgresql-postgresql-0 -- /bin/bash -c "rm -rf /tmp/$file" kubectl cp -n db $file postgresql-postgresql-0:/tmp/ - kubectl exec -n db postgresql-postgresql-0 -- /bin/bash -c "PGPASSWORD=asayerPostgres psql -U postgres -f /tmp/$file" &> "{{ playbook_dir }}"/postgresql_init.log + kubectl exec -n db postgresql-postgresql-0 -- /bin/bash -c "PGPASSWORD=asayerPostgres psql -v ON_ERROR_STOP=1 -U postgres -f /tmp/$file" &> "{{ playbook_dir }}"/postgresql_init.log args: chdir: db/init_dbs/postgresql with_fileglob: diff --git a/scripts/helm/roles/openreplay/templates/alerts.yaml b/scripts/helm/roles/openreplay/templates/alerts.yaml index b28a73a53..b2a91832b 100644 --- a/scripts/helm/roles/openreplay/templates/alerts.yaml +++ b/scripts/helm/roles/openreplay/templates/alerts.yaml @@ -5,7 +5,7 @@ image: {% endif %} env: LICENSE_KEY: "{{ enterprise_edition_license }}" - POSTGRES_STRING: "postgres://{{postgres_db_user}}:{{postgres_db_password}}@{{postgres_endpoint}}:{{postgres_port}}" + POSTGRES_STRING: "postgres://{{postgres_db_user}}:{{postgres_db_password}}@{{postgres_endpoint}}:{{postgres_port}}/{{ postgres_db_name }}" {% if not (docker_registry_username is defined and docker_registry_username and docker_registry_password is defined and docker_registry_password) %} imagePullSecrets: [] diff --git a/scripts/helm/roles/openreplay/templates/assets.yaml b/scripts/helm/roles/openreplay/templates/assets.yaml index 41f898260..740617166 100644 --- a/scripts/helm/roles/openreplay/templates/assets.yaml +++ b/scripts/helm/roles/openreplay/templates/assets.yaml @@ -6,6 +6,7 @@ image: env: AWS_ACCESS_KEY_ID: "{{ minio_access_key }}" AWS_SECRET_ACCESS_KEY: "{{ minio_secret_key }}" + S3_BUCKET_ASSETS: "{{ assets_bucket }}" LICENSE_KEY: "{{ enterprise_edition_license }}" AWS_ENDPOINT: "{{ s3_endpoint }}" AWS_REGION: "{{ aws_region }}" diff --git a/scripts/helm/roles/openreplay/templates/chalice.yaml b/scripts/helm/roles/openreplay/templates/chalice.yaml index 46de7488b..68c1cf9ff 100644 --- a/scripts/helm/roles/openreplay/templates/chalice.yaml +++ b/scripts/helm/roles/openreplay/templates/chalice.yaml @@ -12,12 +12,11 @@ env: S3_SECRET: "{{ minio_secret_key }}" sourcemaps_bucket_key: "{{ minio_access_key }}" sourcemaps_bucket_secret: "{{ minio_secret_key }}" - S3_HOST: "https://{{ domain_name }}" SITE_URL: "https://{{ domain_name }}" jwt_secret: "{{ jwt_secret_key }}" pg_host: "{{ postgres_endpoint }}" pg_port: "{{ postgres_port }}" - pg_dbname: "{{ postgres_port }}" + pg_dbname: "{{ postgres_db_name }}" pg_user: "{{ postgres_db_user }}" pg_password: "{{ postgres_db_password }}" EMAIL_HOST: "{{ email_host }}" @@ -29,8 +28,18 @@ env: EMAIL_SSL_KEY: "{{ email_ssl_key }}" EMAIL_SSL_CERT: "{{ email_ssl_cert }}" EMAIL_FROM: "{{ email_from }}" - AWS_DEFAULT_REGION: "{{ aws_default_region }}" - sessions_region: "{{ aws_default_region }}" + AWS_DEFAULT_REGION: "{{ aws_region }}" + sessions_region: "{{ aws_region }}" + sessions_bucket: "{{ recordings_bucket }}" + sourcemaps_bucket: "{{ sourcemaps_bucket }}" + js_cache_bucket: "{{ assets_bucket }}" + # In case of minio, the instance is running inside kuberntes, + # which is accessible via nginx ingress. +{% if s3_endpoint == "http://minio.db.svc.cluster.local:9000" %} + S3_HOST: "https://{{ domain_name }}" +{% else %} + S3_HOST: "{{ s3_endpoint }}" +{% endif %} {% if env is defined and env.chalice is defined and env.chalice%} {{ env.chalice | to_nice_yaml | trim | indent(2) }} {% endif %} diff --git a/scripts/helm/roles/openreplay/templates/db.yaml b/scripts/helm/roles/openreplay/templates/db.yaml index 7456794c8..bc128593d 100644 --- a/scripts/helm/roles/openreplay/templates/db.yaml +++ b/scripts/helm/roles/openreplay/templates/db.yaml @@ -5,7 +5,7 @@ image: {% endif %} env: LICENSE_KEY: "{{ enterprise_edition_license }}" - POSTGRES_STRING: "postgres://{{ postgres_db_user }}:{{ postgres_password }}@{{ postgres_endpoint }}:{{ postgres_port }}" + POSTGRES_STRING: "postgres://{{ postgres_db_user }}:{{ postgres_db_password }}@{{ postgres_endpoint }}:{{ postgres_port }}/{{ postgres_db_name }}" REDIS_STRING: "{{ redis_endpoint }}" KAFKA_SERVERS: "{{ kafka_endpoint }}" KAFKA_USE_SSL: "{{ kafka_ssl }}" diff --git a/scripts/helm/roles/openreplay/templates/http.yaml b/scripts/helm/roles/openreplay/templates/http.yaml index da7b0979f..a6f9d86b4 100644 --- a/scripts/helm/roles/openreplay/templates/http.yaml +++ b/scripts/helm/roles/openreplay/templates/http.yaml @@ -8,7 +8,7 @@ env: AWS_SECRET_ACCESS_KEY: "{{ minio_secret_key }}" LICENSE_KEY: "{{ enterprise_edition_license }}" AWS_REGION: "{{ aws_region }}" - POSTGRES_STRING: "postgres://{{ postgres_db_user }}:{{ postgres_password }}@{{ postgres_endpoint }}:{{ postgres_port }}" + POSTGRES_STRING: "postgres://{{ postgres_db_user }}:{{ postgres_db_password }}@{{ postgres_endpoint }}:{{ postgres_port }}/{{ postgres_db_name }}" REDIS_STRING: "{{ redis_endpoint }}" KAFKA_SERVERS: "{{ kafka_endpoint }}" KAFKA_USE_SSL: "{{ kafka_ssl }}" diff --git a/scripts/helm/roles/openreplay/templates/integrations.yaml b/scripts/helm/roles/openreplay/templates/integrations.yaml index 9cc8f8b76..f7ea17428 100644 --- a/scripts/helm/roles/openreplay/templates/integrations.yaml +++ b/scripts/helm/roles/openreplay/templates/integrations.yaml @@ -5,7 +5,7 @@ image: {% endif %} env: LICENSE_KEY: "{{ enterprise_edition_license }}" - POSTGRES_STRING: "postgres://{{ postgres_db_user }}:{{ postgres_password }}@{{ postgres_endpoint }}:{{ postgres_port }}" + POSTGRES_STRING: "postgres://{{ postgres_db_user }}:{{ postgres_db_password }}@{{ postgres_endpoint }}:{{ postgres_port }}/{{ postgres_db_name }}" # REDIS_STRING: "{{ redis_endpoint }}" KAFKA_SERVERS: "{{ kafka_endpoint }}" diff --git a/scripts/helm/roles/openreplay/templates/storage.yaml b/scripts/helm/roles/openreplay/templates/storage.yaml index 6a70f3a4c..7a4e080d5 100644 --- a/scripts/helm/roles/openreplay/templates/storage.yaml +++ b/scripts/helm/roles/openreplay/templates/storage.yaml @@ -10,6 +10,8 @@ env: AWS_ENDPOINT: "{{ s3_endpoint }}" AWS_REGION_WEB: "{{ aws_region }}" AWS_REGION_IOS: "{{ aws_region }}" + S3_BUCKET_WEB: "{{ recordings_bucket }}" + S3_BUCKET_IOS: "{{ recordings_bucket }}" REDIS_STRING: "{{ redis_endpoint }}" KAFKA_SERVERS: "{{ kafka_endpoint }}" KAFKA_USE_SSL: "{{ kafka_ssl }}" diff --git a/scripts/helm/vars.yaml b/scripts/helm/vars.yaml index ca0037b27..098c1ad8f 100644 --- a/scripts/helm/vars.yaml +++ b/scripts/helm/vars.yaml @@ -90,6 +90,9 @@ db_resource_override: ## Sane defaults s3_endpoint: "http://minio.db.svc.cluster.local:9000" aws_region: "us-east-1" +assets_bucket: sessions-assets +recordings_bucket: mobs +sourcemaps_bucket: sourcemaps kafka_endpoint: kafka.db.svc.cluster.local:9042 kafka_ssl: false postgres_endpoint: postgresql.db.svc.cluster.local diff --git a/tracker/tracker/package-lock.json b/tracker/tracker/package-lock.json index 8d1c160b5..e1d647441 100644 --- a/tracker/tracker/package-lock.json +++ b/tracker/tracker/package-lock.json @@ -1,6 +1,6 @@ { "name": "@openreplay/tracker", - "version": "3.4.1", + "version": "3.4.4", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -293,11 +293,6 @@ "to-fast-properties": "^2.0.0" } }, - "@medv/finder": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@medv/finder/-/finder-2.0.0.tgz", - "integrity": "sha512-gV4jOsGpiWNDGd8Dw7tod1Fc9Gc7StaOT4oZ/6srHRWtsHU+HYWzmkYsa3Qy/z0e9tY1WpJ9wWdBFGskfbzoug==" - }, "@nodelib/fs.scandir": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 41934717c..808aef967 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "3.4.4", + "version": "3.4.5", "keywords": [ "logging", "replay" @@ -38,7 +38,6 @@ "typescript": "^4.3.4" }, "dependencies": { - "@medv/finder": "^2.0.0", "error-stack-parser": "^2.0.6" }, "engines": { diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index b50bb5731..d23094dfc 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -131,7 +131,7 @@ export default class App { }); } if(this.options.__debug_log) { - warn("OpenReplay errror: ", context, e) + warn("OpenReplay error: ", context, e) } } @@ -153,9 +153,13 @@ export default class App { } } - addCommitCallback(cb: CommitCallback): void { + attachCommitCallback(cb: CommitCallback): void { this.commitCallbacks.push(cb) } + // @Depricated (TODO: remove in 3.5.*) + addCommitCallback(cb: CommitCallback): void { + this.attachCommitCallback(cb) + } safe void>(fn: T): T { diff --git a/tracker/tracker/src/main/app/observer.ts b/tracker/tracker/src/main/app/observer.ts index 493c7aaac..3fad12c60 100644 --- a/tracker/tracker/src/main/app/observer.ts +++ b/tracker/tracker/src/main/app/observer.ts @@ -411,11 +411,13 @@ export default class Observer { private iframeObservers: Observer[] = []; private handleIframe(iframe: HTMLIFrameElement): void { + let context: Window | null = null const handle = () => { - const context = iframe.contentWindow as Window | null const id = this.app.nodes.getID(iframe) - if (!context || id === undefined) { return } - + if (id === undefined) { return } + if (iframe.contentWindow === context) { return } + context = iframe.contentWindow as Window | null; + if (!context) { return } const observer = new Observer(this.app, this.options, context) this.iframeObservers.push(observer) observer.observeIframe(id, context) diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index ca0dd9208..b9fdad6cc 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -23,11 +23,11 @@ import { Options as AppOptions } from './app'; import { Options as ConsoleOptions } from './modules/console'; import { Options as ExceptionOptions } from './modules/exception'; import { Options as InputOptions } from './modules/input'; -import { Options as MouseOptions } from './modules/mouse'; import { Options as PerformanceOptions } from './modules/performance'; import { Options as TimingOptions } from './modules/timing'; + export type Options = Partial< - AppOptions & ConsoleOptions & ExceptionOptions & InputOptions & MouseOptions & PerformanceOptions & TimingOptions + AppOptions & ConsoleOptions & ExceptionOptions & InputOptions & PerformanceOptions & TimingOptions > & { projectID?: number; // For the back compatibility only (deprecated) projectKey: string; @@ -98,7 +98,7 @@ export default class API { Exception(this.app, options); Img(this.app); Input(this.app, options); - Mouse(this.app, options); + Mouse(this.app); Timing(this.app, options); Performance(this.app, options); Scroll(this.app); diff --git a/tracker/tracker/src/main/modules/console.ts b/tracker/tracker/src/main/modules/console.ts index 34be0264a..e625961a7 100644 --- a/tracker/tracker/src/main/modules/console.ts +++ b/tracker/tracker/src/main/modules/console.ts @@ -110,7 +110,7 @@ export default function (app: App, opts: Partial): void { return; } - const sendConsoleLog = app.safe((level: string, args: any[]): void => + const sendConsoleLog = app.safe((level: string, args: unknown[]): void => app.send(new ConsoleLog(level, printf(args))), ); @@ -121,18 +121,36 @@ export default function (app: App, opts: Partial): void { app.attachStartCallback(reset); app.ticker.attach(reset, 33, false); - options.consoleMethods.forEach((method) => { - if (consoleMethods.indexOf(method) === -1) { - console.error(`OpenReplay: unsupported console method "${method}"`); - return; - } - const fn = (console as any)[method]; - (console as any)[method] = function (...args: any[]): void { - fn.apply(this, args); - if (n++ > options.consoleThrottling) { + const patchConsole = (console: Console) => + options.consoleMethods!.forEach((method) => { + if (consoleMethods.indexOf(method) === -1) { + console.error(`OpenReplay: unsupported console method "${method}"`); return; } - sendConsoleLog(method, args); - }; - }); + const fn = (console as any)[method]; + (console as any)[method] = function (...args: unknown[]): void { + fn.apply(this, args); + if (n++ > options.consoleThrottling) { + return; + } + sendConsoleLog(method, args); + }; + }); + patchConsole(window.console); + + app.nodes.attachNodeCallback(node => { + if (node instanceof HTMLIFrameElement) { + let context = node.contentWindow + if (context) { + patchConsole((context as (Window & typeof globalThis)).console) + } + app.attachEventListener(node, "load", () => { + if (node.contentWindow !== context) { + context = node.contentWindow + patchConsole((context as (Window & typeof globalThis)).console) + } + }) + } + + }) } diff --git a/tracker/tracker/src/main/modules/longtasks.ts b/tracker/tracker/src/main/modules/longtasks.ts index e74110b71..c7515c88f 100644 --- a/tracker/tracker/src/main/modules/longtasks.ts +++ b/tracker/tracker/src/main/modules/longtasks.ts @@ -47,5 +47,5 @@ export default function (app: App): void { const observer: PerformanceObserver = new PerformanceObserver((list) => list.getEntries().forEach(longTask), ); - observer.observe({ entryTypes: ['longtask'], buffered: true }); + observer.observe({ entryTypes: ['longtask'] }); } \ No newline at end of file diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 40bcbeb61..8a808f4bf 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -1,10 +1,30 @@ -import type { Options as FinderOptions } from '../vendors/finder/finder'; -import { finder } from '../vendors/finder/finder'; import { normSpaces, hasOpenreplayAttribute, getLabelAttribute } from '../utils'; import App from '../app'; import { MouseMove, MouseClick } from '../../messages'; import { getInputLabel } from './input'; +function _getSelector(target: Element): string { + let el: Element | null = target + let selector: string | null = null + do { + if (el.id) { + return `#${el.id}` + (selector ? ` > ${selector}` : '') + } + selector = + el.className.split(' ') + .map(cn => cn.trim()) + .filter(cn => cn !== '') + .reduce((sel, cn) => `${sel}.${cn}`, el.tagName.toLowerCase()) + + (selector ? ` > ${selector}` : ''); + if (el === document.body) { + return selector + } + el = el.parentElement + } while (el !== document.body && el !== null) + return selector +} + +//TODO: fix (typescript doesn't allow work when the guard is inside the function) function getTarget(target: EventTarget | null): Element | null { if (target instanceof Element) { return _getTarget(target); @@ -72,26 +92,11 @@ function getTargetLabel(target: Element): string { return ''; } -interface HeatmapsOptions { - finder: FinderOptions, -} - -export interface Options { - heatmaps: boolean | HeatmapsOptions; -} - -export default function (app: App, opts: Partial): void { - const options: Options = Object.assign( - { - heatmaps: false // { - // finder: { - // threshold: 5, - // maxNumberOfTries: 600, - // }, - // }, - }, - opts, - ); +export default function (app: App): void { + // const options: Options = Object.assign( + // {}, + // opts, + // ); let mousePositionX = -1; let mousePositionY = -1; @@ -115,9 +120,7 @@ export default function (app: App, opts: Partial): void { const selectorMap: {[id:number]: string} = {}; function getSelector(id: number, target: Element): string { - if (options.heatmaps === false) { return '' } - return selectorMap[id] = selectorMap[id] || - finder(target, options.heatmaps === true ? undefined : options.heatmaps.finder); + return selectorMap[id] = selectorMap[id] || _getSelector(target); } app.attachEventListener(