WIP: Added option to save files in archive

pb/api.proto

@@ -69,6 +69,7 @@ message Job {
   string sshPrivateKey = 21;
   bool sshClone = 22;
   string branch = 23;
+  repeated string archive = 24;
 }
 
 message Command {

server/api/api.go

@@ -109,8 +109,10 @@ func (r Router) Handler() http.Handler {
 
 	cors := cors.New(corsOpts)
 	router.Use(cors.Handler)
-
 	router.Mount("/api/v1", r.apiRouter())
+
+	router.Get("/archive/{id}/", build.HandleArchive(r.Jobs, r.Config))
+
 	router.Get("/ws", ws.UpstreamHandler(r.Config.Websocket.Addr))
 	router.Get("/badge/{token}", badge.HandleBadge(r.Builds))
 	router.Mount("/uploads", r.fileServer())
@@ -247,6 +249,7 @@ func (r Router) workersRouter() *chi.Mux {
 		router.Post("/auth", worker.HandleAuth(r.Workers, r.Config, r.WS.App))
 		router.Post("/cache", worker.HandleUploadCache(r.Config))
 		router.Get("/cache", worker.HandleDownloadCache(r.Config))
+		router.Post("/archive", worker.HandleUploadArchive(r.Config))
 	})
 
 	return router

server/api/build/archive.go

@@ -0,0 +1,33 @@
+package build
+
+import (
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/bleenco/abstruse/server/api/render"
+	"github.com/bleenco/abstruse/server/config"
+	"github.com/bleenco/abstruse/server/core"
+	"github.com/go-chi/chi"
+)
+
+func HandleArchive(jobs core.JobStore, config *config.Config) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, err := strconv.Atoi(chi.URLParam(r, "id"))
+		if err != nil {
+			render.BadRequestError(w, err.Error())
+			return
+		}
+
+		file := path.Join(config.DataDir, "archive", strconv.Itoa(id), r.URL.Path[len(strconv.Itoa(id))+9:])
+		f, err := os.Open(file)
+		if err != nil {
+			render.NotFoundError(w, err.Error())
+			return
+		}
+		defer f.Close()
+		http.ServeContent(w, r, "", time.Now(), f)
+	}
+}

server/api/worker/upload_archive.go

@@ -0,0 +1,59 @@
+package worker
+
+import (
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/bleenco/abstruse/server/api/render"
+	"github.com/bleenco/abstruse/server/config"
+	"github.com/mholt/archiver/v3"
+)
+
+// HandleUploadArchive returns http.handlerFunc that writes JSON encoded
+// result about uploading cache to the http response body.
+func HandleUploadArchive(config *config.Config) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		r.ParseMultipartForm(1000 << 20)
+
+		src, handler, err := r.FormFile("file")
+		if err != nil {
+			render.InternalServerError(w, err.Error())
+			return
+		}
+		defer src.Close()
+		os.MkdirAll(filepath.Join(config.DataDir, "archive"), 0750)
+		filePath := filepath.Join(config.DataDir, "archive", handler.Filename)
+		dst, err := os.Create(filePath)
+		if err != nil {
+			render.InternalServerError(w, err.Error())
+			return
+		}
+		defer dst.Close()
+
+		if _, err := io.Copy(dst, src); err != nil {
+			render.InternalServerError(w, err.Error())
+			return
+		}
+
+		if err = treatArchive(filePath, config.DataDir); err != nil {
+			render.InternalServerError(w, err.Error())
+			return
+		}
+		render.JSON(w, http.StatusOK, render.BoolResponse{Status: true})
+	}
+}
+
+func treatArchive(filePath, datadir string) error {
+	idSplit := strings.Split(strings.ReplaceAll(filePath, "/", "."), ".")
+	//                              The job id
+	targetDir := path.Join(datadir, "archive", idSplit[len(idSplit)-2])
+	os.MkdirAll(targetDir, 0750)
+	log.Println(filePath, "/", targetDir)
+	defer os.RemoveAll(filePath)
+	return archiver.Unarchive(filePath, targetDir)
+}

server/core/job.go

@@ -16,6 +16,7 @@ type (
 		Log       string     `gorm:"size:16777216" json:"-"`
 		Stage     string     `json:"stage"`
 		Cache     string     `json:"cache"`
+		Archive   string     `json:"string"`
 		Build     *Build     `gorm:"preload:false" json:"build,omitempty"`
 		BuildID   uint       `json:"buildID"`
 		Timestamp

server/parser/parser.go

@@ -31,6 +31,7 @@ type RepoConfig struct {
 	AfterDeploy   []string       `yaml:"after_deploy"`
 	AfterScript   []string       `yaml:"after_script"`
 	Cache         []string       `yaml:"cache"`
+	Archive       []string       `yaml:"archive"`
 }
 
 // MatrixConfig defines structure for matrix job config in .abstruse.yml file.
@@ -54,15 +55,17 @@ type JobConfig struct {
 	Title    string           `json:"title"`
 	Commands *api.CommandList `json:"commands"`
 	Cache    []string         `json:"cache"`
+	Archive  []string         `json:"archive"`
 }
 
 // ConfigParser defines repository configuration parser.
 type ConfigParser struct {
-	Raw    string
-	Branch string
-	Parsed RepoConfig
-	Env    []string
-	Mount  []string
+	Raw     string
+	Branch  string
+	Parsed  RepoConfig
+	Env     []string
+	Mount   []string
+	Archive []string
 }
 
 // NewConfigParser returns new config parser instance.
@@ -94,6 +97,7 @@ func (c *ConfigParser) Parse() ([]*JobConfig, error) {
 	if len(c.Parsed.Matrix) > 0 {
 		for _, item := range c.Parsed.Matrix {
 			job := &JobConfig{}
+			job.Archive = c.Parsed.Archive
 
 			// set image
 			if item.Image != "" {
@@ -123,7 +127,6 @@ func (c *ConfigParser) Parse() ([]*JobConfig, error) {
 			}
 			job.Commands = c.generateCommands()
 			job.Cache = c.Parsed.Cache
-
 			jobs = append(jobs, job)
 		}
 	} else {
@@ -135,6 +138,7 @@ func (c *ConfigParser) Parse() ([]*JobConfig, error) {
 			Title:    strings.Join(c.Parsed.Script, " "),
 			Commands: c.generateCommands(),
 			Cache:    c.Parsed.Cache,
+			Archive:  c.Parsed.Archive,
 		}
 		if job.Image == "" {
 			return jobs, fmt.Errorf("image not specified")
@@ -152,6 +156,7 @@ func (c *ConfigParser) Parse() ([]*JobConfig, error) {
 			Title:    strings.Join(c.Parsed.Deploy, " "),
 			Commands: c.generateDeployCommands(),
 			Cache:    c.Parsed.Cache,
+			Archive:  c.Parsed.Archive,
 		}
 		if job.Image == "" {
 			return jobs, fmt.Errorf("image not specified")

server/scheduler/scheduler.go

@@ -330,6 +330,7 @@ func (s *scheduler) startJob(job *core.Job, worker *core.Worker) {
 		Action:        pb.Job_JobStart,
 		WorkerId:      worker.ID,
 		Cache:         strings.Split(job.Cache, ","),
+		Archive:       strings.Split(job.Archive, ","),
 		Mount:         strings.Split(job.Mount, ","),
 		SshPrivateKey: job.Build.Repository.SSHPrivateKey,
 		SshClone:      job.Build.Repository.UseSSH,

server/store/build/build.go

@@ -206,6 +206,7 @@ func (s buildStore) GenerateBuild(repo *core.Repository, base *core.GitHook) ([]
 			BuildID:  build.ID,
 			Mount:    strings.Join(mnts, ","),
 			Cache:    strings.Join(j.Cache, ","),
+			Archive:  strings.Join(j.Archive, ","),
 		}
 		if err := s.jobs.Create(job); err != nil {
 			return nil, 0, err
@@ -332,6 +333,7 @@ func (s buildStore) TriggerBuild(opts core.TriggerBuildOpts) ([]*core.Job, error
 			Stage:    j.Stage,
 			BuildID:  build.ID,
 			Cache:    strings.Join(j.Cache, ","),
+			Archive:  strings.Join(j.Archive, ","),
 		}
 		if err := s.jobs.Create(job); err != nil {
 			return nil, err

web/abstruse/proxy.conf.json

@@ -12,6 +12,10 @@
     "target": "http://localhost",
     "secure": false
   },
+  "/archive": {
+    "target": "http://localhost",
+    "secure": false
+  },
   "/uploads": {
     "target": "http://localhost",
     "secure": false

worker/archive/archive.go

@@ -0,0 +1,95 @@
+package archive
+
+import (
+	"archive/tar"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	api "github.com/bleenco/abstruse/pb"
+	"github.com/bleenco/abstruse/pkg/fs"
+	gzip "github.com/klauspost/pgzip"
+)
+
+func SaveArchive(job *api.Job, dir string) (string, error) {
+	fileName := fmt.Sprintf("%d.tgz", job.GetId())
+	out := filepath.Join(dir, fileName)
+
+	if fs.Exists(out) {
+		if err := os.RemoveAll(out); err != nil {
+			return out, err
+		}
+	}
+
+	return out, createArchive(job.GetArchive(), out)
+}
+
+func createArchive(folders []string, outPath string) error {
+	out, err := os.Create(outPath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	gw, err := gzip.NewWriterLevel(out, gzip.BestSpeed)
+	if err != nil {
+		return err
+	}
+	defer gw.Close()
+
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+
+	for _, folder := range folders {
+		folder = filepath.Join(filepath.Dir(outPath), folder)
+		if err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+			if info.IsDir() {
+				return nil
+			}
+
+			if err != nil {
+				return err
+			}
+
+			link := ""
+			if info.Mode()&os.ModeSymlink != 0 {
+				link, err = os.Readlink(path)
+				if err != nil {
+					return err
+				}
+			}
+
+			header, err := tar.FileInfoHeader(info, link)
+			if err != nil {
+				return err
+			}
+
+			header.Name = strings.TrimPrefix(path, fmt.Sprintf("%s/", filepath.Dir(outPath)))
+
+			if err := tw.WriteHeader(header); err != nil {
+				return err
+			}
+
+			switch header.Typeflag {
+			case tar.TypeLink, tar.TypeSymlink, tar.TypeChar, tar.TypeBlock, tar.TypeDir, tar.TypeFifo:
+			default:
+				file, err := os.Open(path)
+				if err != nil {
+					return err
+				}
+
+				if _, err := io.Copy(tw, file); err != nil {
+					return err
+				}
+			}
+
+			return nil
+		}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}