diff --git a/backend/internal/assets/cacher/cacher.go b/backend/internal/assets/cacher/cacher.go index 2b7f42575..69a7e5149 100644 --- a/backend/internal/assets/cacher/cacher.go +++ b/backend/internal/assets/cacher/cacher.go @@ -143,6 +143,7 @@ func (c *cacher) cacheURL(t *Task) { if contentType == "" { contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path)) } + contentEncoding := res.Header.Get("Content-Encoding") // Skip html file (usually it's a CDN mock for 404 error) if strings.HasPrefix(contentType, "text/html") { @@ -159,7 +160,7 @@ func (c *cacher) cacheURL(t *Task) { // TODO: implement in streams start = time.Now() - err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, objectstorage.NoCompression) + err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, contentEncoding, objectstorage.NoCompression) if err != nil { metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true) c.Errors <- errors.Wrap(err, t.urlContext) diff --git a/backend/internal/canvas-handler/service.go b/backend/internal/canvas-handler/service.go index 57c8580c9..d7e42a965 100644 --- a/backend/internal/canvas-handler/service.go +++ b/backend/internal/canvas-handler/service.go @@ -154,7 +154,7 @@ func (v *ImageStorage) sendToS3(payload interface{}) { if err != nil { v.log.Fatal(task.ctx, "failed to read canvas archive: %s", err) } - if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.Zstd); err != nil { + if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.NoContentEncoding, objectstorage.Zstd); err != nil { v.log.Fatal(task.ctx, "failed to upload canvas to storage: %s", err) } v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start)) diff --git a/backend/internal/screenshot-handler/service.go b/backend/internal/screenshot-handler/service.go index 4b5243567..dbd5f2382 100644 --- a/backend/internal/screenshot-handler/service.go +++ b/backend/internal/screenshot-handler/service.go @@ -159,7 +159,7 @@ func (v *ImageStorage) sendToS3(payload interface{}) { if err != nil { v.log.Fatal(task.ctx, "failed to read replay file: %s", err) } - if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.Zstd); err != nil { + if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.NoContentEncoding, objectstorage.Zstd); err != nil { v.log.Fatal(task.ctx, "failed to upload replay file: %s", err) } v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start)) diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go index 0c008ddde..71e2db2f2 100644 --- a/backend/internal/storage/storage.go +++ b/backend/internal/storage/storage.go @@ -385,7 +385,7 @@ func (s *Storage) uploadSession(payload interface{}) { metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil { s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) } uploadDoms = time.Now().Sub(start).Milliseconds() @@ -398,7 +398,7 @@ func (s *Storage) uploadSession(payload interface{}) { metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil { s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) } uploadDome = time.Now().Sub(start).Milliseconds() @@ -411,7 +411,7 @@ func (s *Storage) uploadSession(payload interface{}) { metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil { s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) } uploadDev = time.Now().Sub(start).Milliseconds() diff --git a/backend/pkg/integrations/service/service.go b/backend/pkg/integrations/service/service.go index 41c7943d5..34098c148 100644 --- a/backend/pkg/integrations/service/service.go +++ b/backend/pkg/integrations/service/service.go @@ -182,7 +182,7 @@ func (s *serviceImpl) fetchSessionData(provider string, credentials interface{}, func (s *serviceImpl) uploadSessionData(provider string, sessionID uint64, data interface{}) error { key := fmt.Sprintf("%d/%s.logs", sessionID, provider) dataBytes, _ := data.([]byte) - return s.storage.Upload(bytes.NewReader(dataBytes), key, "text/plain", objectstorage.NoCompression) + return s.storage.Upload(bytes.NewReader(dataBytes), key, "text/plain", objectstorage.NoContentEncoding, objectstorage.NoCompression) } func (s *serviceImpl) markSessionData(projectID uint64, provider string, sessionID uint64) error { diff --git a/backend/pkg/objectstorage/objectstorage.go b/backend/pkg/objectstorage/objectstorage.go index a514d70ef..4c1bfcc32 100644 --- a/backend/pkg/objectstorage/objectstorage.go +++ b/backend/pkg/objectstorage/objectstorage.go @@ -14,8 +14,10 @@ const ( Zstd ) +const NoContentEncoding = "" + type ObjectStorage interface { - Upload(reader io.Reader, key string, contentType string, compression CompressionType) error + Upload(reader io.Reader, key string, contentType, contentEncoding string, compression CompressionType) error Get(key string) (io.ReadCloser, error) Exists(key string) bool GetCreationTime(key string) *time.Time diff --git a/backend/pkg/objectstorage/s3/s3.go b/backend/pkg/objectstorage/s3/s3.go index 4997079c5..0f625b4fd 100644 --- a/backend/pkg/objectstorage/s3/s3.go +++ b/backend/pkg/objectstorage/s3/s3.go @@ -67,19 +67,22 @@ func NewS3(cfg *objConfig.ObjectsConfig) (objectstorage.ObjectStorage, error) { }, nil } -func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error { +func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error { cacheControl := "max-age=2628000, immutable, private" - var contentEncoding *string + var encoding *string switch compression { case objectstorage.Gzip: encodeStr := "gzip" - contentEncoding = &encodeStr + encoding = &encodeStr case objectstorage.Brotli: encodeStr := "br" - contentEncoding = &encodeStr + encoding = &encodeStr case objectstorage.Zstd: // Have to ignore contentEncoding for Zstd (otherwise will be an error in browser) } + if contentEncoding != "" { + encoding = &contentEncoding + } _, err := s.uploader.Upload(&s3manager.UploadInput{ Body: reader, @@ -87,7 +90,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c Key: &key, ContentType: &contentType, CacheControl: &cacheControl, - ContentEncoding: contentEncoding, + ContentEncoding: encoding, Tagging: s.fileTag, }) return err diff --git a/backend/pkg/spot/api/handlers.go b/backend/pkg/spot/api/handlers.go index 2b6f66c09..87fa5b6e5 100644 --- a/backend/pkg/spot/api/handlers.go +++ b/backend/pkg/spot/api/handlers.go @@ -99,7 +99,7 @@ func (e *handlersImpl) createSpot(w http.ResponseWriter, r *http.Request) { } previewName := fmt.Sprintf("%d/preview.jpeg", newSpot.ID) - if err = e.objStorage.Upload(bytes.NewReader(previewImage), previewName, "image/jpeg", objectstorage.NoCompression); err != nil { + if err = e.objStorage.Upload(bytes.NewReader(previewImage), previewName, "image/jpeg", objectstorage.NoContentEncoding, objectstorage.NoCompression); err != nil { e.log.Error(r.Context(), "can't upload preview image: %s", err) e.responser.ResponseWithError(e.log, r.Context(), w, http.StatusInternalServerError, errors.New("can't upload preview image"), startTime, r.URL.Path, bodySize) return diff --git a/backend/pkg/spot/transcoder/transcoder.go b/backend/pkg/spot/transcoder/transcoder.go index de8df0005..2770a2f46 100644 --- a/backend/pkg/spot/transcoder/transcoder.go +++ b/backend/pkg/spot/transcoder/transcoder.go @@ -249,7 +249,7 @@ func (t *transcoderImpl) cropSpotVideo(spotID uint64, crop []int, path string) e metrics.RecordCroppedVideoSize(float64(fileInfo.Size())) } - err = t.objStorage.Upload(video, fmt.Sprintf("%d/video.webm", spotID), "video/webm", objectstorage.NoCompression) + err = t.objStorage.Upload(video, fmt.Sprintf("%d/video.webm", spotID), "video/webm", objectstorage.NoContentEncoding, objectstorage.NoCompression) if err != nil { return fmt.Errorf("failed to upload cropped video: %v", err) } @@ -321,7 +321,7 @@ func (t *transcoderImpl) transcodeSpotVideo(spotID uint64, path string) (string, defer chunkFile.Close() key := fmt.Sprintf("%d/%s", spotID, chunk) - err = t.objStorage.Upload(chunkFile, key, "video/mp2t", objectstorage.NoCompression) + err = t.objStorage.Upload(chunkFile, key, "video/mp2t", objectstorage.NoContentEncoding, objectstorage.NoCompression) if err != nil { fmt.Println("Error uploading file:", err) return "", err