From bdd62b812d5d15c0a83632e975be774eea8975b2 Mon Sep 17 00:00:00 2001 From: Alexey Remizov Date: Thu, 5 Mar 2020 06:20:35 +0300 Subject: [PATCH] Build and upload binaries and Linux packages --- .gitignore | 3 +- .gitlab-ci.yml | 152 +++++++++++++++++++++++++++++++++++-------- README.md | 6 +- ci/build_binaries | 18 +++++ ci/build_image | 18 +++++ ci/build_package | 10 +++ ci/gitlab_release | 62 ++++++++++++++++++ ci/upload_package | 3 + ci/validate_version | 26 ++++++++ debian/changelog | 32 +++++++++ debian/compat | 1 + debian/control | 15 +++++ debian/copyright | 24 +++++++ debian/default | 20 ++++++ debian/docs | 1 + debian/postinst | 29 +++++++++ debian/rules | 12 ++++ debian/service | 12 ++++ debian/source/format | 1 + 19 files changed, 415 insertions(+), 30 deletions(-) create mode 100755 ci/build_binaries create mode 100755 ci/build_image create mode 100755 ci/build_package create mode 100755 ci/gitlab_release create mode 100755 ci/upload_package create mode 100755 ci/validate_version create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/default create mode 100644 debian/docs create mode 100644 debian/postinst create mode 100755 debian/rules create mode 100644 debian/service create mode 100644 debian/source/format diff --git a/.gitignore b/.gitignore index fd02e0c..81e2571 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ prometheus-logstash-exporter -.idea +/.idea/ +/venv/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35b8669..63ebc05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,36 +1,134 @@ -variables: - DOCKER_DRIVER: overlay - REPO_NAME: registry.gitlab.com/alxrem/prometheus-logstash-exporter +stages: + - validate + - build + - upload + - announce -build_test_image: - variables: - TAG: $CI_COMMIT_REF_SLUG - image: docker:latest - services: - - docker:dind - stage: build +.release: only: - - branches + - tags except: - - master - when: manual + - master + +.build_package: + stage: build + extends: + - .release script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com - - docker build -t $REPO_NAME:$TAG . - - docker push $REPO_NAME:$TAG + - ci/build_package + artifacts: + paths: + - .result/ + expire_in: 30m -build_release_image: - variables: - TAG: $CI_COMMIT_TAG +.upload_package: + stage: upload + extends: + - .release + image: registry.gitlab.com/alxrem/package_cloud + script: + - ci/upload_package + +validate: + stage: validate + image: alpine + extends: + - .release + script: + - apk -U add git + - ci/validate_version + +build:image: + stage: build image: docker:latest services: - - docker:dind + - docker:19.03.5-dind + except: + - master + script: + - ci/build_image + +build:stretch: + extends: + - .build_package + image: debian:stretch + +upload:stretch: + variables: + DIST: debian/stretch + extends: + - .upload_package + needs: + - build:stretch + +build:buster: + extends: + - .build_package + image: debian:buster + +upload:buster: + variables: + DIST: debian/buster + extends: + - .upload_package + needs: + - build:buster + +build:xenial: + extends: + - .build_package + image: ubuntu:xenial + +upload:xenial: + variables: + DIST: ubuntu/xenial + extends: + - .upload_package + needs: + - build:xenial + +build:bionic: + extends: + - .build_package + image: ubuntu:bionic + +upload:bionic: + variables: + DIST: ubuntu/bionic + extends: + - .upload_package + needs: + - build:bionic + +build:binaries: stage: build - only: - - tags + extends: + - .release + image: golang:alpine + script: + - apk add -U binutils + - ci/build_binaries + artifacts: + paths: + - binaries/ + expire_in: 30m + +announce: + stage: announce + extends: + - .release + image: python:3-alpine + needs: + - build:binaries + - job: upload:stretch + artifacts: false + - job: upload:buster + artifacts: false + - job: upload:xenial + artifacts: false + - job: upload:bionic + artifacts: false script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com - - docker build -t $REPO_NAME:$TAG . - - docker tag $REPO_NAME:$TAG $REPO_NAME:latest - - docker push $REPO_NAME:$TAG - - docker push $REPO_NAME:latest + - pip install requests + - apk -U add git + - ci/gitlab_release diff --git a/README.md b/README.md index 03e1637..6c74bfd 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,12 @@ To see all available configuration flags: Packages -------- -Packages for Debian Stretch and Ubuntu Xenial are available on +Binary builds are available on the [releases page of Gitlab project](https://gitlab.com/alxrem/prometheus-logstash-exporter/-/releases). + +Packages for latest Debian and Ubuntu releases are available on [PackageCloud](https://packagecloud.io/alxrem/prometheus-logstash-exporter/). -Docker image is available at [Docker Hub](https://hub.docker.com/r/alxrem/prometheus-logstash-exporter/). +Docker images are available at [Docker Hub](https://hub.docker.com/r/alxrem/prometheus-logstash-exporter/). Pull the latest version with docker pull alxrem/prometheus-logstash-exporter \ No newline at end of file diff --git a/ci/build_binaries b/ci/build_binaries new file mode 100755 index 0000000..602ce8e --- /dev/null +++ b/ci/build_binaries @@ -0,0 +1,18 @@ +#!/bin/sh + +BUILDDIR=/go/src/gitlab.com/$CI_PROJECT_PATH +DEST=`pwd`/binaries +GOARCH=amd64 + +mkdir -p ${DEST}/ +mkdir -p `dirname $BUILDDIR` +ln -s `pwd` $BUILDDIR +cd $BUILDDIR + +for GOOS in windows linux darwin; do + BINARY="${DEST}/${CI_PROJECT_NAME}-${CI_COMMIT_TAG}-${GOOS}-${GOARCH}" + SUM="${BINARY}.sha256" + CGO_ENABLED=0 go build -o $BINARY + strip $BINARY + sha256sum $BINARY | sed -e 's|\(\w\+\s\+\).*/\(.*\)$|\1\2|' > $SUM +done diff --git a/ci/build_image b/ci/build_image new file mode 100755 index 0000000..9d19392 --- /dev/null +++ b/ci/build_image @@ -0,0 +1,18 @@ +#!/bin/sh + +TAG=${CI_COMMIT_TAG:-$CI_COMMIT_REF_SLUG} +docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +docker build -t $CI_REGISTRY_IMAGE:$TAG . +docker push $REPO_NAME:$TAG + +if [ -n "$CI_COMMIT_TAG" ]; then + docker tag $CI_REGISTRY_IMAGE:$TAG $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest + + docker login -u $DOCKERHUB_USER -p $DOCKERHUB_TOKEN + docker tag $CI_REGISTRY_IMAGE:$TAG alxrem/prometheus-logstash-exporter:$TAG + docker push alxrem/prometheus-logstash-exporter:$TAG + + docker tag $CI_REGISTRY_IMAGE:$TAG alxrem/prometheus-logstash-exporter:latest + docker push alxrem/prometheus-logstash-exporter:latest +fi diff --git a/ci/build_package b/ci/build_package new file mode 100755 index 0000000..78a63ff --- /dev/null +++ b/ci/build_package @@ -0,0 +1,10 @@ +#!/bin/sh + +apt-get -qq update +apt-get -qq --no-install-recommends install devscripts equivs + +mk-build-deps -irBt "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -qq -y" +dpkg-buildpackage -uc -us +rm -rf .result && mkdir -p .result +find ../ -maxdepth 1 -type f -exec mv '{}' .result/ ';' +find .result/ diff --git a/ci/gitlab_release b/ci/gitlab_release new file mode 100755 index 0000000..f3e2116 --- /dev/null +++ b/ci/gitlab_release @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import os +import subprocess +from glob import glob +from urllib.parse import urljoin + +import requests + + +def do_release(): + gitlab_token = os.environ["GITLAB_TOKEN"] + version = os.environ["CI_COMMIT_TAG"] + project = os.environ["CI_PROJECT_ID"] + project_url = os.environ["CI_PROJECT_URL"] + headers = {"PRIVATE-TOKEN": gitlab_token} + + # https://docs.gitlab.com/ee/api/projects.html#upload-a-file + uploads_url = \ + "https://gitlab.com/api/v4/projects/{}/uploads".format(project) + + # https://docs.gitlab.com/ee/api/releases/#create-a-release + release_url = \ + "https://gitlab.com/api/v4/projects/{}/releases".format(project) + + binaries = [] + assets_links = [] + for binary in glob("binaries/*"): + files = {"file": open(binary, "rb")} + + r = requests.post(uploads_url, headers=headers, files=files) + if r.status_code >= 400: + print(r.text) + exit(1) + + upload = r.json() + binaries.append(upload["markdown"]) + assets_links.append({ + 'name': upload['alt'], + 'url': urljoin(project_url + '/', upload['url'].lstrip('/')), + }) + + description = \ + subprocess.getoutput("git tag {} -l --format='%(contents:body)'" + .format(version)) + + release = { + "name": version, + "tag_name": version, + "description": description, + 'assets': { + 'links': sorted(assets_links, key=lambda x: x['name'], reverse=True), + } + } + r = requests.post(release_url, headers=headers, json=release) + if r.status_code >= 400: + print(r.text) + exit(1) + + +if __name__ == "__main__": + do_release() diff --git a/ci/upload_package b/ci/upload_package new file mode 100755 index 0000000..fcce1f6 --- /dev/null +++ b/ci/upload_package @@ -0,0 +1,3 @@ +#!/bin/sh + +package_cloud push alxrem/prometheus-logstash-exporter/$DIST .result/*.deb \ No newline at end of file diff --git a/ci/validate_version b/ci/validate_version new file mode 100755 index 0000000..0e1f2ce --- /dev/null +++ b/ci/validate_version @@ -0,0 +1,26 @@ +#!/bin/sh + +SANITIZED_VERSION=$(echo "$CI_COMMIT_TAG" | egrep -ox '[0-9]+\.[0-9]+\.[0-9]+.*') + +if [ -z "$SANITIZED_VERSION" ]; then + echo "Invalid version format \"$CI_COMMIT_TAG\"" + exit 1 +fi + +ANNOTATION=$(git tag $CI_COMMIT_TAG -l --format='%(contents:body)' | tr -d '[[:space:]]') +if [ -z "$ANNOTATION" ]; then + echo "Tag should by annotated" + exit 1 +fi + +CHANGELOG_EXPECTED="prometheus-logstash-exporter ($CI_COMMIT_TAG)" +CHANGELOG_FOUND=$(head -n1 debian/changelog | fgrep -o "$CHANGELOG_EXPECTED") + +if [ "$CHANGELOG_FOUND" != "$CHANGELOG_EXPECTED" ]; then + echo "Version $CHANGELOG_EXPECTED expected" + echo + echo "Found $(head -n1 debian/changelog)" + exit 1 +fi + +exit 0 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..e46451d --- /dev/null +++ b/debian/changelog @@ -0,0 +1,32 @@ +prometheus-logstash-exporter (0.6.1) unstable; urgency=medium + + * Technical release to tag updated CI scripts. + + -- Alexey Remizov Thu, 10 Mar 2020 01:23:50 +0300 + +prometheus-logstash-exporter (0.5.0) unstable; urgency=medium + + * Parse values of type timestamp + Thanks to Michael DOUBEZ + + -- Alexey Remizov Thu, 15 Aug 2019 00:57:47 +0300 + +prometheus-logstash-exporter (0.4.0) unstable; urgency=medium + + * Fixed processing of `filters.patterns_per_field` section + Thanks to Casey Weed + + -- Alexey Remizov Mon, 20 Aug 2018 23:11:00 +0300 + +prometheus-logstash-exporter (0.2.0) unstable; urgency=medium + + * Added support of Logstash 6.x. + Thanks to Guo Xiang Tan + + -- Alexey Remizov Thu, 14 Dec 2017 02:29:59 +0300 + +prometheus-logstash-exporter (0.1.0) unstable; urgency=low + + * Initial release. + + -- Alexey Remizov Tue, 12 Sep 2017 00:49:07 +0300 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..35eb4d0 --- /dev/null +++ b/debian/control @@ -0,0 +1,15 @@ +Source: prometheus-logstash-exporter +Maintainer: Alexey Remizov +Section: utils +Priority: optional +Build-Depends: debhelper (>= 9), dh-systemd, dh-golang, golang (>= 2:1.6) +Standards-Version: 3.9.4 + +Package: prometheus-logstash-exporter +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends} +Suggests: logstash +Description: Prometheus exporter for Logstash metrics + Prometheus exporter for metrics provided by Node Stats API of Logstash. + + diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..f8870f3 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,24 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: prometheus-logstash-exporter +Upstream-Contact: Alexey Remizov +Source: https://gitlab.com/alxrem/prometheus-logstash-exporter + +Files: * +Copyright: 2017 Alexey Remizov +License: Apache-2.0 + +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the full text of the Apache License, Version 2.0 + can be found in the file `/usr/share/common-licenses/Apache-2.0'. diff --git a/debian/default b/debian/default new file mode 100644 index 0000000..eddfcba --- /dev/null +++ b/debian/default @@ -0,0 +1,20 @@ +ARGS="" + +# Usage of ./prometheus-logstash-exporter: +# -log.format value +# Set the log target and format. +# Example: "logger:syslog?appname=bob&local=7" +# or "logger:stdout?json=true" (default "logger:stderr") +# -log.level value +# Only log messages with the given severity or above. +# Valid levels: [debug, info, warn, error, fatal] +# -logstash.host string +# Host address of logstash server. (default "localhost") +# -logstash.port int +# Port of logstash server. (default 9600) +# -logstash.timeout duration +# Timeout to get stats from logstash server. (default 5s) +# -web.listen-address string +# Address to listen on for web interface and telemetry. (default ":9304") +# -web.telemetry-path string +# Path under which to expose metrics. (default "/metrics") diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000..42061c0 --- /dev/null +++ b/debian/docs @@ -0,0 +1 @@ +README.md \ No newline at end of file diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 0000000..d72b9d7 --- /dev/null +++ b/debian/postinst @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e + +case "$1" in + configure|reconfigure) + # Add prometheus user + if ! getent passwd prometheus > /dev/null; then + adduser --quiet --system --no-create-home \ + --group --gecos "Prometheus daemon" prometheus || true + fi + + mkdir -p /var/log/prometheus + chown -R prometheus:prometheus /var/log/prometheus || true + + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + : + ;; + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..c1607da --- /dev/null +++ b/debian/rules @@ -0,0 +1,12 @@ +#!/usr/bin/make -f + +export DH_GOPKG := gitlab.com/alxrem/prometheus-logstash-exporter +export DH_GOLANG_EXCLUDES := vendor + +%: + dh $@ --buildsystem=golang --with=golang,systemd + +override_dh_auto_test: + +override_dh_auto_install: + dh_auto_install -- --no-source diff --git a/debian/service b/debian/service new file mode 100644 index 0000000..ea3a691 --- /dev/null +++ b/debian/service @@ -0,0 +1,12 @@ +[Unit] +Description=Prometheus exporter for Logstash +Documentation=https://prometheus.io/docs/introduction/overview/ + +[Service] +Restart=always +User=prometheus +EnvironmentFile=/etc/default/prometheus-logstash-exporter +ExecStart=/usr/bin/prometheus-logstash-exporter $ARGS + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native)