diff --git a/backend/internal/assets/cacher/cacher.go b/backend/internal/assets/cacher/cacher.go index 2b7f42575..69a7e5149 100644 --- a/backend/internal/assets/cacher/cacher.go +++ b/backend/internal/assets/cacher/cacher.go @@ -143,6 +143,7 @@ func (c *cacher) cacheURL(t *Task) { if contentType == "" { contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path)) } + contentEncoding := res.Header.Get("Content-Encoding") // Skip html file (usually it's a CDN mock for 404 error) if strings.HasPrefix(contentType, "text/html") { @@ -159,7 +160,7 @@ func (c *cacher) cacheURL(t *Task) { // TODO: implement in streams start = time.Now() - err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, objectstorage.NoCompression) + err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, contentEncoding, objectstorage.NoCompression) if err != nil { metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true) c.Errors <- errors.Wrap(err, t.urlContext) diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go index 0c008ddde..efec53277 100644 --- a/backend/internal/storage/storage.go +++ b/backend/internal/storage/storage.go @@ -385,7 +385,7 @@ func (s *Storage) uploadSession(payload interface{}) { metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", "", task.compression); err != nil { s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) } uploadDoms = time.Now().Sub(start).Milliseconds() @@ -398,7 +398,7 @@ func (s *Storage) uploadSession(payload interface{}) { metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", "", task.compression); err != nil { s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) } uploadDome = time.Now().Sub(start).Milliseconds() @@ -411,7 +411,7 @@ func (s *Storage) uploadSession(payload interface{}) { metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", "", task.compression); err != nil { s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) } uploadDev = time.Now().Sub(start).Milliseconds() diff --git a/backend/pkg/objectstorage/objectstorage.go b/backend/pkg/objectstorage/objectstorage.go index a514d70ef..afd8333d8 100644 --- a/backend/pkg/objectstorage/objectstorage.go +++ b/backend/pkg/objectstorage/objectstorage.go @@ -15,7 +15,7 @@ const ( ) type ObjectStorage interface { - Upload(reader io.Reader, key string, contentType string, compression CompressionType) error + Upload(reader io.Reader, key string, contentType, contentEncoding string, compression CompressionType) error Get(key string) (io.ReadCloser, error) Exists(key string) bool GetCreationTime(key string) *time.Time diff --git a/backend/pkg/objectstorage/s3/s3.go b/backend/pkg/objectstorage/s3/s3.go index 4997079c5..0f625b4fd 100644 --- a/backend/pkg/objectstorage/s3/s3.go +++ b/backend/pkg/objectstorage/s3/s3.go @@ -67,19 +67,22 @@ func NewS3(cfg *objConfig.ObjectsConfig) (objectstorage.ObjectStorage, error) { }, nil } -func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error { +func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error { cacheControl := "max-age=2628000, immutable, private" - var contentEncoding *string + var encoding *string switch compression { case objectstorage.Gzip: encodeStr := "gzip" - contentEncoding = &encodeStr + encoding = &encodeStr case objectstorage.Brotli: encodeStr := "br" - contentEncoding = &encodeStr + encoding = &encodeStr case objectstorage.Zstd: // Have to ignore contentEncoding for Zstd (otherwise will be an error in browser) } + if contentEncoding != "" { + encoding = &contentEncoding + } _, err := s.uploader.Upload(&s3manager.UploadInput{ Body: reader, @@ -87,7 +90,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c Key: &key, ContentType: &contentType, CacheControl: &cacheControl, - ContentEncoding: contentEncoding, + ContentEncoding: encoding, Tagging: s.fileTag, }) return err diff --git a/ee/backend/pkg/objectstorage/azure/azure.go b/ee/backend/pkg/objectstorage/azure/azure.go index 0dab51388..984c80849 100644 --- a/ee/backend/pkg/objectstorage/azure/azure.go +++ b/ee/backend/pkg/objectstorage/azure/azure.go @@ -50,16 +50,19 @@ func NewStorage(cfg *config.ObjectsConfig) (objectstorage.ObjectStorage, error) }, nil } -func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error { +func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error { cacheControl := "max-age=2628000, immutable, private" - var contentEncoding *string + var encoding *string switch compression { case objectstorage.Gzip: gzipStr := "gzip" - contentEncoding = &gzipStr + encoding = &gzipStr case objectstorage.Brotli: gzipStr := "br" - contentEncoding = &gzipStr + encoding = &gzipStr + } + if contentEncoding != "" { + encoding = &contentEncoding } // Remove leading slash to avoid empty folder creation if strings.HasPrefix(key, "/") { @@ -68,7 +71,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c _, err := s.client.UploadStream(context.Background(), s.container, key, reader, &azblob.UploadStreamOptions{ HTTPHeaders: &blob.HTTPHeaders{ BlobCacheControl: &cacheControl, - BlobContentEncoding: contentEncoding, + BlobContentEncoding: encoding, BlobContentType: &contentType, }, Tags: s.tags,