From f0a0378a7a0ea2390b9b14e2671f5fb4920cabf7 Mon Sep 17 00:00:00 2001 From: Evgeni Gordeev Date: Wed, 15 Apr 2020 17:01:21 -0500 Subject: [PATCH] Closes #43. --- README.md | 1 + internal/config/config.go | 122 +++++++++++++++++---------------- internal/config/config_test.go | 57 +++++++-------- internal/controllers/s3.go | 6 +- internal/service/amazon-s3.go | 14 ++++ internal/service/client.go | 1 + 6 files changed, 113 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index d730fa6..c0c69d7 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ AWS_API_ENDPOINT | The endpoint for AWS API for local development. | INDEX_DOCUMENT | Name of your index document. | | index.html DIRECTORY_LISTINGS | List files when a specified URL ends with /. | | false DIRECTORY_LISTINGS_FORMAT | Configures directory listing to be `html` (spider parsable) | | - +DIRECTORY_LISTINGS_CHECK_INDEX | Check for `INDEX_DOCUMENT` in the folder before listing files | | false HTTP_CACHE_CONTROL | Overrides S3's HTTP `Cache-Control` header. | | S3 Object metadata HTTP_EXPIRES | Overrides S3's HTTP `Expires` header. | | S3 Object metadata BASIC_AUTH_USER | User for basic authentication. | | - diff --git a/internal/config/config.go b/internal/config/config.go index 6e80c32..5103c43 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,35 +17,36 @@ func init() { } type config struct { // nolint - AwsRegion string // AWS_REGION - AwsAPIEndpoint string // AWS_API_ENDPOINT - S3Bucket string // AWS_S3_BUCKET - S3KeyPrefix string // AWS_S3_KEY_PREFIX - IndexDocument string // INDEX_DOCUMENT - DirectoryListing bool // DIRECTORY_LISTINGS - DirListingFormat string // DIRECTORY_LISTINGS_FORMAT - HTTPCacheControl string // HTTP_CACHE_CONTROL (max-age=86400, no-cache ...) - HTTPExpires string // HTTP_EXPIRES (Thu, 01 Dec 1994 16:00:00 GMT ...) - BasicAuthUser string // BASIC_AUTH_USER - BasicAuthPass string // BASIC_AUTH_PASS - Port string // APP_PORT - Host string // APP_HOST - AccessLog bool // ACCESS_LOG - SslCert string // SSL_CERT_PATH - SslKey string // SSL_KEY_PATH - StripPath string // STRIP_PATH - ContentEncoding bool // CONTENT_ENCODING - CorsAllowOrigin string // CORS_ALLOW_ORIGIN - CorsAllowMethods string // CORS_ALLOW_METHODS - CorsAllowHeaders string // CORS_ALLOW_HEADERS - CorsMaxAge int64 // CORS_MAX_AGE - HealthCheckPath string // HEALTHCHECK_PATH - AllPagesInDir bool // GET_ALL_PAGES_IN_DIR - MaxIdleConns int // MAX_IDLE_CONNECTIONS - IdleConnTimeout time.Duration // IDLE_CONNECTION_TIMEOUT - DisableCompression bool // DISABLE_COMPRESSION - InsecureTLS bool // Disables TLS validation on request endpoints. - JwtSecretKey string // JWT_SECRET_KEY + AwsRegion string // AWS_REGION + AwsAPIEndpoint string // AWS_API_ENDPOINT + S3Bucket string // AWS_S3_BUCKET + S3KeyPrefix string // AWS_S3_KEY_PREFIX + IndexDocument string // INDEX_DOCUMENT + DirectoryListing bool // DIRECTORY_LISTINGS + DirListingFormat string // DIRECTORY_LISTINGS_FORMAT + DirListingCheckIndex bool // DIRECTORY_LISTINGS_CHECK_INDEX + HTTPCacheControl string // HTTP_CACHE_CONTROL (max-age=86400, no-cache ...) + HTTPExpires string // HTTP_EXPIRES (Thu, 01 Dec 1994 16:00:00 GMT ...) + BasicAuthUser string // BASIC_AUTH_USER + BasicAuthPass string // BASIC_AUTH_PASS + Port string // APP_PORT + Host string // APP_HOST + AccessLog bool // ACCESS_LOG + SslCert string // SSL_CERT_PATH + SslKey string // SSL_KEY_PATH + StripPath string // STRIP_PATH + ContentEncoding bool // CONTENT_ENCODING + CorsAllowOrigin string // CORS_ALLOW_ORIGIN + CorsAllowMethods string // CORS_ALLOW_METHODS + CorsAllowHeaders string // CORS_ALLOW_HEADERS + CorsMaxAge int64 // CORS_MAX_AGE + HealthCheckPath string // HEALTHCHECK_PATH + AllPagesInDir bool // GET_ALL_PAGES_IN_DIR + MaxIdleConns int // MAX_IDLE_CONNECTIONS + IdleConnTimeout time.Duration // IDLE_CONNECTION_TIMEOUT + DisableCompression bool // DISABLE_COMPRESSION + InsecureTLS bool // Disables TLS validation on request endpoints. + JwtSecretKey string // JWT_SECRET_KEY } // Setup configurations with environment variables @@ -66,6 +67,10 @@ func Setup() { if b, err := strconv.ParseBool(os.Getenv("DIRECTORY_LISTINGS")); err == nil { directoryListings = b } + directoryListingsCheckIndex := false + if b, err := strconv.ParseBool(os.Getenv("DIRECTORY_LISTINGS_CHECK_INDEX")); err == nil { + directoryListingsCheckIndex = b + } accessLog := false if b, err := strconv.ParseBool(os.Getenv("ACCESS_LOG")); err == nil { accessLog = b @@ -99,35 +104,36 @@ func Setup() { insecureTLS = b } Config = &config{ - AwsRegion: region, - AwsAPIEndpoint: os.Getenv("AWS_API_ENDPOINT"), - S3Bucket: os.Getenv("AWS_S3_BUCKET"), - S3KeyPrefix: os.Getenv("AWS_S3_KEY_PREFIX"), - IndexDocument: indexDocument, - DirectoryListing: directoryListings, - DirListingFormat: os.Getenv("DIRECTORY_LISTINGS_FORMAT"), - HTTPCacheControl: os.Getenv("HTTP_CACHE_CONTROL"), - HTTPExpires: os.Getenv("HTTP_EXPIRES"), - BasicAuthUser: os.Getenv("BASIC_AUTH_USER"), - BasicAuthPass: os.Getenv("BASIC_AUTH_PASS"), - Port: port, - Host: os.Getenv("APP_HOST"), - AccessLog: accessLog, - SslCert: os.Getenv("SSL_CERT_PATH"), - SslKey: os.Getenv("SSL_KEY_PATH"), - StripPath: os.Getenv("STRIP_PATH"), - ContentEncoding: contentEncoding, - CorsAllowOrigin: os.Getenv("CORS_ALLOW_ORIGIN"), - CorsAllowMethods: os.Getenv("CORS_ALLOW_METHODS"), - CorsAllowHeaders: os.Getenv("CORS_ALLOW_HEADERS"), - CorsMaxAge: corsMaxAge, - HealthCheckPath: os.Getenv("HEALTHCHECK_PATH"), - AllPagesInDir: allPagesInDir, - MaxIdleConns: maxIdleConns, - IdleConnTimeout: idleConnTimeout, - DisableCompression: disableCompression, - InsecureTLS: insecureTLS, - JwtSecretKey: os.Getenv("JWT_SECRET_KEY"), + AwsRegion: region, + AwsAPIEndpoint: os.Getenv("AWS_API_ENDPOINT"), + S3Bucket: os.Getenv("AWS_S3_BUCKET"), + S3KeyPrefix: os.Getenv("AWS_S3_KEY_PREFIX"), + IndexDocument: indexDocument, + DirectoryListing: directoryListings, + DirListingCheckIndex: directoryListingsCheckIndex, + DirListingFormat: os.Getenv("DIRECTORY_LISTINGS_FORMAT"), + HTTPCacheControl: os.Getenv("HTTP_CACHE_CONTROL"), + HTTPExpires: os.Getenv("HTTP_EXPIRES"), + BasicAuthUser: os.Getenv("BASIC_AUTH_USER"), + BasicAuthPass: os.Getenv("BASIC_AUTH_PASS"), + Port: port, + Host: os.Getenv("APP_HOST"), + AccessLog: accessLog, + SslCert: os.Getenv("SSL_CERT_PATH"), + SslKey: os.Getenv("SSL_KEY_PATH"), + StripPath: os.Getenv("STRIP_PATH"), + ContentEncoding: contentEncoding, + CorsAllowOrigin: os.Getenv("CORS_ALLOW_ORIGIN"), + CorsAllowMethods: os.Getenv("CORS_ALLOW_METHODS"), + CorsAllowHeaders: os.Getenv("CORS_ALLOW_HEADERS"), + CorsMaxAge: corsMaxAge, + HealthCheckPath: os.Getenv("HEALTHCHECK_PATH"), + AllPagesInDir: allPagesInDir, + MaxIdleConns: maxIdleConns, + IdleConnTimeout: idleConnTimeout, + DisableCompression: disableCompression, + InsecureTLS: insecureTLS, + JwtSecretKey: os.Getenv("JWT_SECRET_KEY"), } // Proxy log.Printf("[config] Proxy to %v", Config.S3Bucket) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c1cd509..afd3d09 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -10,34 +10,35 @@ import ( func defaultConfig() *config { return &config{ - AwsRegion: "", - AwsAPIEndpoint: "", - S3Bucket: "", - S3KeyPrefix: "", - IndexDocument: "index.html", - DirectoryListing: false, - DirListingFormat: "", - HTTPCacheControl: "", - HTTPExpires: "", - BasicAuthUser: "", - BasicAuthPass: "", - Port: "80", - Host: "", - AccessLog: false, - SslCert: "", - SslKey: "", - StripPath: "", - ContentEncoding: true, - CorsAllowOrigin: "", - CorsAllowMethods: "", - CorsAllowHeaders: "", - CorsMaxAge: int64(600), - HealthCheckPath: "", - AllPagesInDir: false, - MaxIdleConns: 150, - IdleConnTimeout: time.Duration(10) * time.Second, - DisableCompression: true, - InsecureTLS: false, + AwsRegion: "", + AwsAPIEndpoint: "", + S3Bucket: "", + S3KeyPrefix: "", + IndexDocument: "index.html", + DirectoryListing: false, + DirListingCheckIndex: false, + DirListingFormat: "", + HTTPCacheControl: "", + HTTPExpires: "", + BasicAuthUser: "", + BasicAuthPass: "", + Port: "80", + Host: "", + AccessLog: false, + SslCert: "", + SslKey: "", + StripPath: "", + ContentEncoding: true, + CorsAllowOrigin: "", + CorsAllowMethods: "", + CorsAllowHeaders: "", + CorsMaxAge: int64(600), + HealthCheckPath: "", + AllPagesInDir: false, + MaxIdleConns: 150, + IdleConnTimeout: time.Duration(10) * time.Second, + DisableCompression: true, + InsecureTLS: false, } } diff --git a/internal/controllers/s3.go b/internal/controllers/s3.go index 7243e7c..5f1444f 100644 --- a/internal/controllers/s3.go +++ b/internal/controllers/s3.go @@ -58,8 +58,10 @@ func AwsS3(w http.ResponseWriter, r *http.Request) { // Ends with / -> listing or index.html if strings.HasSuffix(path, "/") { if c.DirectoryListing { - s3listFiles(w, r, client, c.S3Bucket, c.S3KeyPrefix+path) - return + if !c.DirListingCheckIndex || !client.S3exists(c.S3Bucket, c.S3KeyPrefix+path+c.IndexDocument) { + s3listFiles(w, r, client, c.S3Bucket, c.S3KeyPrefix+path) + return + } } path += c.IndexDocument } diff --git a/internal/service/amazon-s3.go b/internal/service/amazon-s3.go index 62d0341..ab344f3 100644 --- a/internal/service/amazon-s3.go +++ b/internal/service/amazon-s3.go @@ -16,6 +16,20 @@ func (c client) S3get(bucket, key string, rangeHeader *string) (*s3.GetObjectOut return s3.New(c.Session).GetObjectWithContext(c.Context, req) } +// S3exists returns true if a specified key exists in Amazon S3 +func (c client) S3exists(bucket, key string) bool { + req := &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + + output, err := s3.New(c.Session).HeadObject(req) + if err != nil { + return false + } + return *output.ContentLength > 0 +} + // S3listObjects returns a list of s3 objects func (c client) S3listObjects(bucket, prefix string) (*s3.ListObjectsOutput, error) { req := &s3.ListObjectsInput{ diff --git a/internal/service/client.go b/internal/service/client.go index 5d547c4..e0f9e63 100644 --- a/internal/service/client.go +++ b/internal/service/client.go @@ -10,6 +10,7 @@ import ( // AWS is a service to interact with original AWS services type AWS interface { S3get(bucket, key string, rangeHeader *string) (*s3.GetObjectOutput, error) + S3exists(bucket, key string) bool S3listObjects(bucket, prefix string) (*s3.ListObjectsOutput, error) }