Update build and release systems

This diff removes the build_tarballs.sh and the rpm/Makefile files and
moves their logic into the main Makefile.

The build system places binary files in the build directory. The release
system copy those files over to the release directory, preparing it for
travis to pick up.

Examples:

make binary_deb binary_rpm VERSION=1.0
make clean binary_tar VERSION=1.0 GOOS=linux GOARCH=amd64
make distclean release VERSION=1.0

Deleted the debian/copyright file because it was outdated; I think this
could be automated with debmake but didn't spend time on it.

There's other minor changes like removing trailing space from files,
updating and adding documentation.

Re-add copyright notice to vulndb/schema.go.

.travis.yml

@@ -10,28 +10,25 @@ services:
  mysql
 
 before_install:
-- sudo apt-get install -y rpm build-essential debhelper dh-make fakeroot
+- sudo apt-get install -y rpm build-essential debhelper dh-make fakeroot zip
 - mysql -e 'CREATE DATABASE vulndb;'
 
 env:
 - MYSQL_TEST_DSN=root@/vulndb
 
 script:
-- go get -v -u ./...
+- go get -v -u -d ./...
 - go test -v ./...
 
 before_deploy:
-- export VERSION=${TRAVIS_TAG:1}
-- chmod +x build_tarballs.sh && ./build_tarballs.sh
-- make -C rpm/
-- dpkg-buildpackage -rfakeroot -uc -us && mv ../*.deb release
+- make release VERSION=${TRAVIS_TAG:1}
 
 deploy:
  provider: releases
  api_key:
  secure: hnofbIf34EiluDy1u288ZVu1CM7f8MxKY8T40Tkj6dj6CBJAtEscm6AQiXP2v2TVLK0mOAtpZ8gK6oGRE6jVmKxrLZNKhuL1E3XNfi5poED8kW0hjjXBCWMG5tLrXCXk7d5qYMMVTD8/eB5XBJMrfiC8GcjuD0QgzqhkQiJdpBnMZj5NamL0JR6Tht2turSglwWAQ9E0GdIaHYpHwN4IJ2wkh0kBoR7uJrIU9F5STOO0OqgJ8LZ9YwCtljgz9XvJt3W6Xds+AH0w/MtfhZucJXG20vCYaTMxdFKoQxitol/YISIOD/P4MhFBzmf2O2Cw5X517owAb++wkfYL+E2NEhVilZlKkTTHBNzCRzO85GGsDCBv91MSDTykZwu047/VWvxxIpGPTC7bQnYFcgWTc3d93pBERRL1ppfzNlU/CPIebLj5KPCCkSzb1RTEy8gOL2aoKzqImv/428P+FVaaibbM0M87OM3iWOe1yGRXUVQtc2CMNdNukr7/p4HFaptPyBu9OjnNsqjsI+yS1flqrDpFbkBV5ZDUhy0tPyu59m9l34qrrGJ1nJdiZvcNgAaC8VEeZ42OUuvNtl/77Wx1X7cYU8Iqz/zz/QWLqkwljQCjoXPEqIBvVyqzn+eeeRNzGDpzbxFf3MMhx7zE7QVk8M+M//SIGFMeyTHswriMkZk=
  file_glob: true
- file: release/**/*
+ file: release/*
  skip_cleanup: true
  on:
  tags: true

Makefile

@@ -1,38 +1,154 @@
-# Go parameters
-GOCMD=go
-GOBUILD=$(GOCMD) build
-GOCLEAN=$(GOCMD) clean
-GOGET=$(GOCMD) get
-CPE2CVE=cpe2cve
-CSV2CPE=csv2cpe
-NVDSYNC=nvdsync
-RPM2CPE=rpm2cpe
-
-NAME=nvdtools
-PKG=$(NAME)-$(VERSION)
-TGZ=$(PKG).tar.gz
-
-all: build 
-build: 
- $(GOBUILD) -o $(CPE2CVE) ./cmd/$(CPE2CVE)/cpe2cve.go
- $(GOBUILD) -o $(CSV2CPE) ./cmd/$(CSV2CPE)/csv2cpe.go
- $(GOBUILD) -o $(NVDSYNC) ./cmd/$(NVDSYNC)/main.go
- $(GOBUILD) -o $(RPM2CPE) ./cmd/$(RPM2CPE)/rpm2cpe.go
-
-clean: 
- $(GOCLEAN)
- rm -f $(CPE2CVE)
- rm -f $(CSV2CPE)
- rm -f $(NVDSYNC)
- rm -f $(RPM2CPE)
-
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+NAME = nvdtools
+VERSION = tip
+
+TOOLS = \
+ cpe2cve \
+ csv2cpe \
+ fireeye2nvd \
+ flexera2nvd \
+ nvdsync \
+ rpm2cpe \
+ rustsec2nvd \
+ vulndb
+
+DOCS = \
+ CODE_OF_CONDUCT.md \
+ CONTRIBUTING.md \
+ HOWTO.md \
+ LICENSE \
+ README.md
+
+GO = go
+GOOS = $(shell $(GO) env GOOS)
+GOARCH = $(shell $(GO) env GOARCH)
+
+TAR = tar
+ZIP = zip
+INSTALL = install
+
+# Compile all tools.
+all: $(TOOLS)
+
+# Compile TOOLS to ./build/bin/$tool using GOOS and GOARCH.
+$(TOOLS):
+ GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build $(GOFLAGS) -o ./build/bin/$@ ./cmd/$@
+
+# Check/fetch all dependencies.
+deps:
+ GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) get -v -d ./...
+
+# install installs tools and documentation.
+# The install target is used by rpm and deb builders.
 install:
- install -d $(DESTDIR)/usr/bin
- install -p -m 0755 ./cpe2cve $(DESTDIR)/usr/bin/cpe2cve
- install -p -m 0755 ./csv2cpe $(DESTDIR)/usr/bin/csv2cpe
- install -p -m 0755 ./nvdsync $(DESTDIR)/usr/bin/nvdsync
- install -p -m 0755 ./rpm2cpe $(DESTDIR)/usr/bin/rpm2cpe 
-
-archive:
- touch $(TGZ)
- tar czf $(TGZ) --exclude=$(TGZ) --transform s/$(NAME)/$(PKG)/ ../$(NAME)
+ # tools
+ $(INSTALL) -d $(DESTDIR)/usr/bin
+ for tool in $(TOOLS); do $(INSTALL) -p -m 0755 ./build/bin/$$tool $(DESTDIR)/usr/bin/$$tool; done
+ # docs
+ $(INSTALL) -d $(DESTDIR)/usr/share/doc/nvdtools
+ for doc in $(DOCS); do $(INSTALL) -p -m 0644 $$doc $(DESTDIR)/usr/share/doc/nvdtools/$$doc; done
+
+DIST_NAME = $(NAME)-$(VERSION)
+DIST_DIR = build/$(DIST_NAME)
+
+# binary_dist creates a local binary distribution in DIST_DIR.
+binary_dist: $(TOOLS)
+ mkdir -p $(DIST_DIR)/doc
+ cp $(DOCS) $(DIST_DIR)/doc
+ mv build/bin $(DIST_DIR)/bin
+
+# binary_tar creates tarball of binary distribution.
+binary_tar: binary_dist
+ mkdir -p build/tgz
+ cd build && $(TAR) czf tgz/$(DIST_NAME)-$(GOOS)-$(GOARCH).tar.gz $(DIST_NAME)
+ rm -rf $(DIST_DIR)
+
+# binary_zip creates zip of binary distribution.
+binary_zip: binary_dist
+ mkdir -p build/zip
+ cd build && $(ZIP) -r zip/$(DIST_NAME)-$(GOOS)-$(GOARCH).zip $(DIST_NAME)
+ rm -rf $(DIST_DIR)
+
+# binary_deb creates debian package.
+#
+# Requires GOPATH and dependencies available to compile nvdtools.
+# Must set version to build: make binary_deb VERSION=1.0
+binary_deb:
+ VERSION=$(VERSION) dpkg-buildpackage -rfakeroot -uc -us
+ mkdir -p build/deb
+ mv ../$(NAME)*.deb build/deb/
+
+# archive_tar creates tarball of the source code.
+archive_tar:
+ mkdir -p build/tgz
+ $(TAR) czf build/tgz/$(DIST_NAME).tar.gz \
+ --exclude=build \
+ --exclude=release \
+ --exclude=.git \
+ --exclude=.travis.yml \
+ --transform s/./$(DIST_NAME)/ \
+ .
+
+# binary_rpm creates rpm package.
+#
+# Requires GOPATH and dependencies available to compile nvdtools.
+# Must set version to build: make binary_rpm VERSION=1.0
+binary_rpm: archive_tar
+ mkdir -p build/rpm/SOURCES
+ mv build/tgz/$(DIST_NAME).tar.gz build/rpm/SOURCES/
+ rpmbuild -ba \
+ --define="_topdir $(PWD)/build/rpm" \
+ --define="_version $(VERSION)" \
+ rpm/nvdtools.spec
+
+# release_tar creates tarball releases.
+release_tar:
+ mkdir -p release
+ make deps binary_tar GOOS=darwin GOARCH=amd64
+ make deps binary_tar GOOS=freebsd GOARCH=amd64
+ make deps binary_tar GOOS=linux GOARCH=amd64
+ make deps binary_tar GOOS=linux GOARCH=arm64
+ mv build/tgz/*.tar.gz release
+
+# release_zip creates zip releases.
+release_zip:
+ mkdir -p release
+ make deps binary_zip GOOS=windows GOARCH=386
+ make deps binary_zip GOOS=windows GOARCH=amd64
+ mv build/zip/*.zip release
+
+# release_deb creates debian releases.
+release_deb: binary_deb
+ mkdir -p release
+ mv build/deb/*.deb release
+
+# release_rpm creates rpm releases.
+release_rpm: binary_rpm
+ mkdir -p release
+ mv build/rpm/RPMS/*/*.rpm release
+
+# release creates all release packages.
+# Example: make distclean release VERSION=1.0
+release: release_deb release_rpm release_tar release_zip
+
+# Removes build related files.
+clean:
+ rm -rf build
+
+distclean: clean
+ rm -rf release
+
+.PHONY: $(TOOLS)

debian/README.Debian

@@ -1 +1 @@
-A set of tools to work with the feeds (vulnerabilities, CPE dictionary etc.) distributed by National Vulnerability Database (NVD)
+A set of tools to work with the feeds (vulnerabilities, CPE dictionary etc.) distributed by National Vulnerability Database (NVD).

debian/control

@@ -1,7 +1,7 @@
 Source: nvdtools
 Section: utils
 Priority: extra
-Maintainer: Alexandre Fiori <fiorix@fb.com> 
+Maintainer: Alexandre Fiori <fiorix@fb.com>
 Build-Depends: debhelper (>=9)
 Standards-Version: 3.9.6
 Homepage: https://github.com/facebookincubator/nvdtools/
@@ -10,4 +10,4 @@ Package: nvdtools
 Architecture: any
 Multi-Arch: foreign
 Depends: ${misc:Depends}, ${shlibs:Depends}
-Description: A set of tools to work with the feeds (vulnerabilities, CPE dictionary etc.) distributed by National Vulnerability Database (NVD)
+Description: A set of tools to work with the feeds (vulnerabilities, CPE dictionary etc.) distributed by National Vulnerability Database (NVD).