From 78ea3c90de75722df1d0ffc095ab288e5c0cbc38 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Wed, 23 Dec 2020 18:34:05 +0100 Subject: [PATCH] Implement package upload to our storage backend on Digital Ocean (#110) --- ci/package-lock.json | 94 +++++++++++++++++++++++++++++++++++++++ ci/package.json | 1 + ci/src/PackageUpload.purs | 55 +++++++++++++++++++++++ ci/src/S3.js | 48 ++++++++++++++++++++ ci/src/S3.purs | 83 ++++++++++++++++++++++++++++++++++ 5 files changed, 281 insertions(+) create mode 100644 ci/src/PackageUpload.purs create mode 100644 ci/src/S3.js create mode 100644 ci/src/S3.purs diff --git a/ci/package-lock.json b/ci/package-lock.json index 140668c2b..76e2e55b6 100644 --- a/ci/package-lock.json +++ b/ci/package-lock.json @@ -116,11 +116,42 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.23.tgz", "integrity": "sha512-Z4U8yDAl5TFkmYsZdFPdjeMa57NOvnaf1tljHzhouaPEp7LCj2JKkejpI1ODviIAQuW4CcQmxkQ77rnLsOOoKw==" }, + "aws-sdk": { + "version": "2.814.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.814.0.tgz", + "integrity": "sha512-empd1m/J/MAkL6d9OeRpmg9thobULu0wk4v8W3JToaxGi2TD7PIdvE6yliZKyOVAdJINhBWEBhxR4OUIHhcGbQ==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "before-after-hook": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.0.tgz", "integrity": "sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==" }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "1.5.1", + "ieee754": "1.1.13", + "isarray": "1.0.0" + } + }, "cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -153,6 +184,11 @@ "once": "1.4.0" } }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "execa": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", @@ -175,6 +211,11 @@ "pump": "3.0.0" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "is-plain-object": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-3.0.1.tgz", @@ -185,11 +226,21 @@ "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "macos-release": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.4.0.tgz", @@ -249,6 +300,21 @@ "once": "1.4.0" } }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, "semver": { "version": "7.3.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", @@ -285,6 +351,20 @@ "os-name": "3.1.0" } }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -310,6 +390,20 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/xhr2/-/xhr2-0.2.0.tgz", "integrity": "sha512-BDtiD0i2iKPK/S8OAZfpk6tyzEDnKKSjxWHcMBVmh+LuqJ8A32qXTyOx+TVOg2dKvq6zGBq2sgKPkEeRs1qTRA==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": "1.2.1", + "xmlbuilder": "9.0.7" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" } } } diff --git a/ci/package.json b/ci/package.json index a9758ddca..452ad9c51 100644 --- a/ci/package.json +++ b/ci/package.json @@ -13,6 +13,7 @@ "license": "MIT", "dependencies": { "@octokit/rest": "^18.0.0", + "aws-sdk": "^2.814.0", "semver": "^7.3.2", "xhr2": "^0.2.0" } diff --git a/ci/src/PackageUpload.purs b/ci/src/PackageUpload.purs new file mode 100644 index 000000000..b4692b87b --- /dev/null +++ b/ci/src/PackageUpload.purs @@ -0,0 +1,55 @@ +module PackageUpload where + +import Prelude + +import Data.Array as Array +import Effect (Effect) +import Effect.Aff as Aff +import Effect.Class.Console (log) +import Node.FS.Aff as FS +import S3 as S3 + +type PackageInfo = + { name :: String + , version :: String + , revision :: Int + } + +type Path = String + +upload :: PackageInfo -> Path -> Effect Unit +upload { name, version, revision } path = Aff.launchAff_ $ do + -- first check that the file path exists + fileContent <- do + fileExists <- FS.exists path + if fileExists + then FS.readFile path + else Aff.throwError $ Aff.error $ "File doesn't exist: " <> show path + -- connect to the bucket + let bucket = "purescript-registry" + log $ "Connecting to the bucket " <> show bucket + s3 <- S3.connect "ams3.digitaloceanspaces.com" bucket + -- check that the file for that version and revision is there + let filename + = name <> "/" + <> version + <> (if revision == 0 then "" else "_r" <> show revision) + <> ".tar.gz" + publishedPackages <- map _.key <$> S3.listObjects s3 { prefix: name <> "/" } + if Array.elem filename publishedPackages + -- if the release is already there we crash + then Aff.throwError $ Aff.error $ "The package " <> show filename <> " already exists" + -- if it's not, we upload it with public read permission + else do + log $ "Uploading release to the bucket: " <> show filename + let putParams = { key: filename, body: fileContent, acl: S3.PublicRead } + void $ S3.putObject s3 putParams + log "Done." + +{- + +main :: Effect Unit +main = do + upload { name: "aff", version: "5.1.2", revision: 0 } "../examples/aff/v5.1.2.json" + +-} diff --git a/ci/src/S3.js b/ci/src/S3.js new file mode 100644 index 000000000..d8f996068 --- /dev/null +++ b/ci/src/S3.js @@ -0,0 +1,48 @@ +const AWS = require('aws-sdk'); + +if ("SPACES_KEY" in process.env && "SPACES_SECRET" in process.env) { + ; +} else { + console.log('Please set SPACES_KEY and SPACES_SECRET envvars'); + process.exit(1); +} + +exports.connectImpl = function(endpoint) { + return function() { + const spacesEndpoint = new AWS.Endpoint(endpoint); + const s3 = new AWS.S3({ + endpoint: spacesEndpoint, + accessKeyId: process.env.SPACES_KEY, + secretAccessKey: process.env.SPACES_SECRET + }); + return s3; + }; +}; + +exports.listObjectsImpl = function(s3, params) { + return function() { + return new Promise(function(resolve, reject) { + s3.listObjectsV2(params, function(err, data) { + if (err) { + reject(err); + } else { + resolve(data['Contents']); + } + }); + }); + }; +}; + +exports.putObjectImpl = function(s3, params) { + return function() { + return new Promise(function(resolve, reject) { + s3.putObject(params, function(err, data) { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); + }; +}; diff --git a/ci/src/S3.purs b/ci/src/S3.purs new file mode 100644 index 000000000..21298929d --- /dev/null +++ b/ci/src/S3.purs @@ -0,0 +1,83 @@ +module S3 ( + connect, + listObjects, + putObject, + S3, + Space, + ACL(..) +) where + +import Prelude + +import Control.Promise (Promise) +import Control.Promise as Promise +import Data.Function.Uncurried (Fn1, Fn2, runFn1, runFn2) +import Data.JSDate (JSDate) +import Data.Traversable (for) +import Effect (Effect) +import Effect.Aff (Aff) +import Effect.Class (liftEffect) +import Node.Buffer (Buffer) + + +foreign import data S3 :: Type + +type Space = { conn :: S3, bucket :: String } + +foreign import connectImpl :: Fn1 String (Effect S3) +connect :: String -> String -> Aff Space +connect region bucket = do + conn <- liftEffect $ runFn1 connectImpl region + pure { bucket, conn } + + +-- Add more params as needed: +-- https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property +type JSListParams = + { "Bucket" :: String + , "Prefix" :: String + } + +type JSListResponse = + { "Key" :: String + , "LastModified" :: JSDate + , "ETag" :: String + , "Size" :: Int + , "StorageClass" :: String + } + +type ListParams = { prefix :: String } +type ListResponse = { key :: String, size :: Int, eTag :: String } + +foreign import listObjectsImpl :: Fn2 S3 JSListParams (Effect (Promise (Array JSListResponse))) +listObjects :: Space -> ListParams -> Aff (Array ListResponse) +listObjects space params = do + let jsParams = { "Bucket": space.bucket, "Prefix": params.prefix } + (jsObjs :: Array JSListResponse) <- Promise.toAffE (runFn2 listObjectsImpl space.conn jsParams) + for jsObjs \obj -> pure { key: obj."Key", size: obj."Size", eTag: obj."ETag" } -- TODO: pull more props if needed + + +-- Add more params as needed: +-- https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property +type JSPutParams = + { "Bucket" :: String + , "Key" :: String + , "Body" :: Buffer -- TODO: the SDK also accepts a string here, but we don't need that right now + , "ACL" :: String -- SDK supports more, but DO only supports "private" and "public-read" + } + +type JSPutResponse = { "ETag" :: String } + +data ACL = Private | PublicRead +type PutParams = { key :: String, body :: Buffer, acl :: ACL } +type PutResponse = { eTag :: String } + +foreign import putObjectImpl :: Fn2 S3 JSPutParams (Effect (Promise JSPutResponse)) +putObject :: Space -> PutParams -> Aff PutResponse +putObject space params = do + let jsACL = case params.acl of + Private -> "private" + PublicRead -> "public-read" + let jsParams = { "Bucket": space.bucket, "Key": params.key, "Body": params.body, "ACL": jsACL } + res <- Promise.toAffE (runFn2 putObjectImpl space.conn jsParams) + pure { eTag: res."ETag" }