Implement package upload to our storage backend on Digital Ocean (#110)

ci/package.json

@@ -13,6 +13,7 @@
   "license": "MIT",
   "dependencies": {
     "@octokit/rest": "^18.0.0",
+    "aws-sdk": "^2.814.0",
     "semver": "^7.3.2",
     "xhr2": "^0.2.0"
   }

ci/src/PackageUpload.purs

@@ -0,0 +1,55 @@
+module PackageUpload where
+
+import Prelude
+
+import Data.Array as Array
+import Effect (Effect)
+import Effect.Aff as Aff
+import Effect.Class.Console (log)
+import Node.FS.Aff as FS
+import S3 as S3
+
+type PackageInfo =
+  { name :: String
+  , version :: String
+  , revision :: Int
+  }
+
+type Path = String
+
+upload :: PackageInfo -> Path -> Effect Unit
+upload { name, version, revision } path = Aff.launchAff_ $ do
+  -- first check that the file path exists
+  fileContent <- do
+    fileExists <- FS.exists path
+    if fileExists
+    then FS.readFile path
+    else Aff.throwError $ Aff.error $ "File doesn't exist: " <> show path
+  -- connect to the bucket
+  let bucket = "purescript-registry"
+  log $ "Connecting to the bucket " <> show bucket
+  s3 <- S3.connect "ams3.digitaloceanspaces.com" bucket
+  -- check that the file for that version and revision is there
+  let filename
+         = name <> "/"
+        <> version
+        <> (if revision == 0 then "" else "_r" <> show revision)
+        <> ".tar.gz"
+  publishedPackages <- map _.key <$> S3.listObjects s3 { prefix: name <> "/" }
+  if Array.elem filename publishedPackages
+  -- if the release is already there we crash
+  then Aff.throwError $ Aff.error $ "The package " <> show filename <> " already exists"
+  -- if it's not, we upload it with public read permission
+  else do
+    log $ "Uploading release to the bucket: " <> show filename
+    let putParams = { key: filename, body: fileContent, acl: S3.PublicRead }
+    void $ S3.putObject s3 putParams
+    log "Done."
+
+{-
+
+main :: Effect Unit
+main = do
+  upload { name: "aff", version: "5.1.2", revision: 0 } "../examples/aff/v5.1.2.json"
+
+-}

ci/src/S3.js

@@ -0,0 +1,48 @@
+const AWS = require('aws-sdk');
+
+if ("SPACES_KEY" in process.env && "SPACES_SECRET" in process.env) {
+  ;
+} else {
+  console.log('Please set SPACES_KEY and SPACES_SECRET envvars');
+  process.exit(1);
+}
+
+exports.connectImpl = function(endpoint) {
+  return function() {
+    const spacesEndpoint = new AWS.Endpoint(endpoint);
+    const s3 = new AWS.S3({
+      endpoint: spacesEndpoint,
+      accessKeyId: process.env.SPACES_KEY,
+      secretAccessKey: process.env.SPACES_SECRET
+    });
+    return s3;
+  };
+};
+
+exports.listObjectsImpl = function(s3, params) {
+  return function() {
+    return new Promise(function(resolve, reject) {
+      s3.listObjectsV2(params, function(err, data) {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(data['Contents']);
+        }
+      });
+    });
+  };
+};
+
+exports.putObjectImpl = function(s3, params) {
+  return function() {
+    return new Promise(function(resolve, reject) {
+      s3.putObject(params, function(err, data) {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(data);
+        }
+      });
+    });
+  };
+};

ci/src/S3.purs

@@ -0,0 +1,83 @@
+module S3 (
+  connect,
+  listObjects,
+  putObject,
+  S3,
+  Space,
+  ACL(..)
+) where
+
+import Prelude
+
+import Control.Promise (Promise)
+import Control.Promise as Promise
+import Data.Function.Uncurried (Fn1, Fn2, runFn1, runFn2)
+import Data.JSDate (JSDate)
+import Data.Traversable (for)
+import Effect (Effect)
+import Effect.Aff (Aff)
+import Effect.Class (liftEffect)
+import Node.Buffer (Buffer)
+
+
+foreign import data S3 :: Type
+
+type Space = { conn :: S3, bucket :: String }
+
+foreign import connectImpl :: Fn1 String (Effect S3)
+connect :: String -> String -> Aff Space
+connect region bucket = do
+  conn <- liftEffect $ runFn1 connectImpl region
+  pure { bucket, conn }
+
+
+-- Add more params as needed:
+-- https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property
+type JSListParams =
+  { "Bucket" :: String
+  , "Prefix" :: String
+  }
+
+type JSListResponse =
+  { "Key" :: String
+  , "LastModified" :: JSDate
+  , "ETag" :: String
+  , "Size" :: Int
+  , "StorageClass" :: String
+  }
+
+type ListParams = { prefix :: String }
+type ListResponse = { key :: String, size :: Int, eTag :: String }
+
+foreign import listObjectsImpl :: Fn2 S3 JSListParams (Effect (Promise (Array JSListResponse)))
+listObjects :: Space -> ListParams -> Aff (Array ListResponse)
+listObjects space params = do
+  let jsParams = { "Bucket": space.bucket, "Prefix": params.prefix }
+  (jsObjs :: Array JSListResponse) <- Promise.toAffE (runFn2 listObjectsImpl space.conn jsParams)
+  for jsObjs \obj -> pure { key: obj."Key", size: obj."Size", eTag: obj."ETag" } -- TODO: pull more props if needed
+
+
+-- Add more params as needed:
+-- https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
+type JSPutParams =
+  { "Bucket" :: String
+  , "Key" :: String
+  , "Body" :: Buffer -- TODO: the SDK also accepts a string here, but we don't need that right now
+  , "ACL" :: String -- SDK supports more, but DO only supports "private" and "public-read"
+  }
+
+type JSPutResponse = { "ETag" :: String }
+
+data ACL = Private | PublicRead
+type PutParams = { key :: String,  body :: Buffer, acl :: ACL }
+type PutResponse = { eTag :: String }
+
+foreign import putObjectImpl :: Fn2 S3 JSPutParams (Effect (Promise JSPutResponse))
+putObject :: Space -> PutParams -> Aff PutResponse
+putObject space params = do
+  let jsACL = case params.acl of
+        Private -> "private"
+        PublicRead -> "public-read"
+  let jsParams = { "Bucket": space.bucket, "Key": params.key, "Body": params.body, "ACL": jsACL }
+  res <- Promise.toAffE (runFn2 putObjectImpl space.conn jsParams)
+  pure { eTag: res."ETag" }