Merge pull request #133 from larsewi/CFE-3990

CFE-3990: Prompt user which bundle to run

cfbs/build.py

@@ -1,4 +1,5 @@
 import os
+import glob
 import logging as log
 from cfbs.utils import (
     canonify,
@@ -120,26 +121,17 @@ def _perform_build_step(module, step, max_length):
         merged = read_json(defjson)
         if not merged:
             merged = {}
-        if "classes" not in merged:
-            merged["classes"] = {}
-        if "services_autorun_bundles" not in merged["classes"]:
-            merged["classes"]["services_autorun_bundles"] = ["any"]
-        inputs = []
-        for root, dirs, files in os.walk(src):
+        for root, _, files in os.walk(src):
             for f in files:
-                if f.endswith(".cf"):
-                    inputs.append(os.path.join(dstarg, f))
-                    cp(os.path.join(root, f), os.path.join(destination, dstarg, f))
-                elif f == "def.json":
+                if f == "def.json":
                     extra = read_json(os.path.join(root, f))
                     if extra:
                         merged = merge_json(merged, extra)
                 else:
-                    cp(os.path.join(root, f), os.path.join(destination, dstarg, f))
-        if "inputs" in merged:
-            merged["inputs"].extend(inputs)
-        else:
-            merged["inputs"] = inputs
+                    s = os.path.join(root, f)
+                    d = os.path.join(destination, dstarg, root[len(src) :], f)
+                    log.debug("Copying '%s' to '%s'" % (s, d))
+                    cp(s, d)
         write_json(defjson, merged)
     elif operation == "input":
         src, dst = args
@@ -170,6 +162,43 @@ def _perform_build_step(module, step, max_length):
             merged = extras
         log.debug("Merged def.json: %s", pretty(merged))
         write_json(dst, merged)
+    elif operation == "policy_files":
+        files = []
+        for file in args:
+            if file.startswith("./"):
+                file = file[2:]
+            if file.endswith(".cf"):
+                files.append(file)
+            elif file.endswith("/"):
+                pattern = "%s**/*.cf" % file
+                files += glob.glob(pattern, recursive=True)
+            else:
+                user_error(
+                    "Unsupported filetype '%s' for build step '%s': "
+                    % (file, operation)
+                    + "Expected directory (*/) of policy file (*.cf)"
+                )
+        files = [os.path.join("services", "cfbs", file) for file in files]
+        print("%s policy_files '%s'" % (prefix, "' '".join(files) if files else ""))
+        augment = {"inputs": files}
+        log.debug("Generated augment: %s" % pretty(augment))
+        path = os.path.join(destination, "def.json")
+        original = read_json(path)
+        log.debug("Original def.json: %s" % pretty(original))
+        merged = merge_json(original, augment) if original else augment
+        log.debug("Merged def.json: %s", pretty(merged))
+        write_json(path, merged)
+    elif operation == "bundles":
+        bundles = args
+        print("%s bundles '%s'" % (prefix, "' '".join(bundles) if bundles else ""))
+        augment = {"vars": {"control_common_bundlesequence_end": bundles}}
+        log.debug("Generated augment: %s" % pretty(augment))
+        path = os.path.join(destination, "def.json")
+        original = read_json(path)
+        log.debug("Original def.json: %s" % pretty(original))
+        merged = merge_json(original, augment) if original else augment
+        log.debug("Merged def.json: %s", pretty(merged))
+        write_json(path, merged)
     else:
         user_error("Unknown build step operation: %s" % operation)
 

cfbs/cfbs_config.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import os
 import copy
+import glob
 import logging as log
 from collections import OrderedDict
 
@@ -10,6 +11,7 @@
     read_file,
     find,
     write_json,
+    load_bundlenames,
 )
 from cfbs.internal_file_management import (
     clone_url_repo,
@@ -49,27 +51,6 @@ class CFBSConfig(CFBSJson):
     def exists(path="./cfbs.json"):
         return os.path.exists(path)
 
-    @staticmethod
-    def validate_added_module(module):
-        """Try to help the user with warnings in appropriate cases"""
-
-        name = module["name"]
-        if name.startswith("./") and name.endswith(".cf"):
-            assert os.path.isfile(name)
-            if not _has_autorun_tag(name):
-                log.warning("No autorun tag found in policy file: '%s'" % name)
-                log.warning("Tag the bundle(s) you want evaluated:")
-                log.warning('  meta: "tags" slist => { "autorun" };')
-            return
-        if name.startswith("./") and name.endswith("/"):
-            assert os.path.isdir(name)
-            policy_files = list(find(name, extension=".cf"))
-            with_autorun = (x for x in policy_files if _has_autorun_tag(x))
-            if any(policy_files) and not any(with_autorun):
-                log.warning("No bundles tagged with autorun found in: '%s'" % name)
-                log.warning("Tag the bundle(s) you want evaluated in .cf policy files:")
-                log.warning('  meta: "tags" slist => { "autorun" };')
-
     @classmethod
     def get_instance(cls, index=None, non_interactive=False):
         if cls.instance is not None:
@@ -126,7 +107,6 @@ def add_with_dependencies(self, module, remote_config=None, dependent=None):
             print("Added module: %s (Dependency of %s)" % (module["name"], dependent))
         else:
             print("Added module: %s" % module["name"])
-        self.validate_added_module(module)
 
     def _add_using_url(
         self,
@@ -225,6 +205,79 @@ def _find_dependencies(self, modules, exclude):
             dependencies += self._find_dependencies(dependencies, exclude)
         return dependencies
 
+    def _add_to_inputs(self, module):
+        name = module["name"]
+        step = "policy_files %s" % name
+        module["steps"].append(step)
+        log.debug("Added build step '%s' for module '%s'" % (step, name))
+
+    def _add_to_bundleseqence(self, module, policy_files):
+        name = module["name"]
+        choices = []
+        first = True
+        prompt_str = "Which bundle should be evaluated (added to bundle sequence)?"
+
+        for file in policy_files:
+            log.debug("Looking for bundles in policy file '%s'" % file)
+            for bundle in load_bundlenames(file):
+                log.debug("Found bundle '%s'" % bundle)
+                choices.append(bundle)
+                prompt_str += "\n%2d. %s:%s" % (len(choices), file, bundle)
+                if first:
+                    prompt_str += " (default)"
+                    first = False
+
+        if not choices:
+            log.warning("Did not find any bundles to add to bundlesequence")
+            return
+
+        choices.append(None)
+        prompt_str += "\n%2d. (None)\n" % (len(choices))
+
+        response = prompt_user(
+            self.non_interactive,
+            prompt_str,
+            [str(i + 1) for i in range(len(choices))],
+            1,
+        )
+        bundle = choices[int(response) - 1]
+        if bundle is None:
+            log.debug("User chose not to add any bundles to the bundlesequence")
+            return
+        log.debug("User chose to add '%s' to the bundlesequence" % bundle)
+
+        step = "bundles %s" % bundle
+        module["steps"].append(step)
+        log.debug("Added build step '%s' for module '%s'" % (step, name))
+
+    def _handle_local_module(self, module):
+        name = module["name"]
+        if not (
+            name.startswith("./")
+            and name.endswith((".cf", "/"))
+            and "local" in module["tags"]
+        ):
+            log.debug("Module '%s' does not appear to be a local module" % name)
+            return
+
+        if name.endswith(".cf"):
+            policy_files = [name]
+        else:
+            pattern = "%s/**/*.cf" % name
+            policy_files = glob.glob(pattern, recursive=True)
+
+        for file in policy_files:
+            if _has_autorun_tag(file):
+                log.warning(
+                    "Found bundle tagged with autorun in local policy file '%s': "
+                    % file
+                    + "Note that the autorun tag is ignored when adding local policy files or subdirectories."
+                )
+                # TODO: Support adding local modules with autorun tag
+
+        self._add_to_inputs(module)
+        self._add_to_bundleseqence(module, policy_files)
+
     def _add_without_dependencies(self, modules):
         assert modules
         assert len(modules) > 0
@@ -236,7 +289,7 @@ def _add_without_dependencies(self, modules):
             if self.index.custom_index != None:
                 module["index"] = self.index.custom_index
             self["build"].append(module)
-            self.validate_added_module(module)
+            self._handle_local_module(module)
 
             added_by = module["added_by"]
             if added_by == "cfbs add":

cfbs/index.py

@@ -14,12 +14,11 @@
 
 
 def _local_module_data_cf_file(module):
-    target = os.path.basename(module)
+    dst = os.path.join("services", "cfbs", module[2:])
     return {
         "description": "Local policy file added using cfbs command line",
         "tags": ["local"],
-        "dependencies": ["autorun"],
-        "steps": ["copy %s services/autorun/%s" % (module, target)],
+        "steps": ["copy %s %s" % (module, dst)],
         "added_by": "cfbs add",
     }
 

cfbs/prompts.py

@@ -12,7 +12,9 @@ def prompt_user(non_interactive, prompt, choices=None, default=None):
 
     prompt_separator = " " if prompt.endswith("?") else ": "
     if choices:
-        assert default is None or str(default) in choices
+        assert (
+            default is None or str(default) in choices
+        ), "'%s' not 'None' and '%s' not in '%s'" % (default, default, choices)
         choices_str = "/".join(
             choice.upper() if choice == str(default) else choice for choice in choices
         )

cfbs/utils.py

@@ -288,3 +288,17 @@ def wrapper(*args, **kwargs):
 def canonify(s: str):
     s = "".join([c if c.isalnum() else "_" for c in s])
     return s
+
+
+def load_bundlenames(file: str):
+    with open(file, "r") as f:
+        policy = f.read()
+    return loads_bundlenames(policy)
+
+
+def loads_bundlenames(policy: str):
+    # The lookbehind only supports fixed length strings
+    policy = re.sub(r"[ \t]+", " ", policy)
+
+    regex = r"(?<=^bundle agent )[a-zA-Z0-9_\200-\377]+"
+    return re.findall(regex, policy, re.MULTILINE)

tests/shell/010_local_add.sh

@@ -8,18 +8,27 @@ rm -rf .git
 
 cfbs --non-interactive init
 cfbs status
-echo '
-bundle agent test_bundle
-{
-  meta:
-    "tags" slist => { "autorun" };
+
+echo 'bundle agent bogus {
   reports:
-    "test";
+      "This is $(this.promise_filename):$(this.bundle)!";
 }
-' > test_policy.cf
-cfbs --non-interactive add ./test_policy.cf
-grep '"name": "autorun"' cfbs.json
-grep '"name": "./test_policy.cf"' cfbs.json
+' > bogus.cf
+
+
+cfbs --non-interactive add ./bogus.cf
+
+grep '"name": "./bogus.cf"' cfbs.json
+grep '"policy_files ./bogus.cf"' cfbs.json
+grep '"bundles bogus"' cfbs.json
+
 cfbs status
 cfbs build
-ls out/masterfiles/services/autorun/test_policy.cf
+
+grep '"inputs"' out/masterfiles/def.json
+grep '"services/cfbs/bogus.cf"' out/masterfiles/def.json
+
+grep '"control_common_bundlesequence_end"' out/masterfiles/def.json
+grep '"bogus"' out/masterfiles/def.json
+
+ls out/masterfiles/services/cfbs/bogus.cf

tests/shell/016_add_folders.sh

@@ -5,63 +5,54 @@ mkdir -p ./tmp/
 cd ./tmp/
 touch cfbs.json && rm cfbs.json
 rm -rf .git
-rm -rf one
-rm -rf two
 
-mkdir one
-echo '
-bundle agent bundle_one
-{
-  meta:
-    "tags" slist => { "autorun" };
+mkdir -p doofus
+echo 'bundle agent doofus {
   reports:
-    "one";
+      "This is $(this.promise_filename):$(this.bundle)!";
 }
-' > one/policy.cf
-echo '{}
-' > one/data.json
+' > doofus/doofus.cf
 
-mkdir two
-mkdir two/three
-echo '
-bundle agent bundle_two
-{
-  meta:
-    "tags" slist => { "autorun" };
+mkdir -p doofus/foo
+echo 'bundle agent foo {
   reports:
-    "two";
+      "This is $(this.promise_filename):$(this.bundle)!";
 }
-' > two/three/policy.cf
+' > doofus/foo/foo.cf
+
+echo '{}
+' > doofus/data.json
+
 echo '{
   "vars": {
     "foo_thing": "awesome"
   }
 }
 
-' > two/three/def.json
-echo 'Hello
-' > two/three/file.txt
+' > doofus/foo/def.json
 
 cfbs --non-interactive init
 cfbs status
 
-cfbs --non-interactive add ./one
-cfbs --non-interactive add ./two/
+cfbs --non-interactive add ./doofus/
 cfbs status
 
-cfbs status | grep "./one/"
-cfbs status | grep "./two/"
-cat cfbs.json | grep "directory ./ services/cfbs/one/"
-cat cfbs.json | grep "directory ./ services/cfbs/two/"
+cfbs status | grep "./doofus/"
+grep '"name": "./doofus/"' cfbs.json
+grep '"directory ./ services/cfbs/doofus/"' cfbs.json
+grep '"policy_files ./doofus/"' cfbs.json
+grep '"bundles doofus"' cfbs.json
 
 cfbs build
 
-ls out/masterfiles/services/cfbs/one
-grep "bundle_one" out/masterfiles/services/cfbs/one/policy.cf
-ls out/masterfiles/services/cfbs/one/data.json
+grep '"inputs"' out/masterfiles/def.json
+grep '"services/cfbs/doofus/doofus.cf"' out/masterfiles/def.json
+grep '"services/cfbs/doofus/foo/foo.cf"' out/masterfiles/def.json
+
+grep '"control_common_bundlesequence_end"' out/masterfiles/def.json
+grep '"doofus"' out/masterfiles/def.json
 
-ls out/masterfiles/services/cfbs/two
-grep "bundle_two" out/masterfiles/services/cfbs/two/policy.cf
-grep "Hello" out/masterfiles/services/cfbs/two/file.txt
+grep '"foo_thing": "awesome"' out/masterfiles/def.json
 
-grep "awesome" out/masterfiles/def.json
+ls out/masterfiles/services/cfbs/doofus/doofus.cf
+ls out/masterfiles/services/cfbs/doofus/foo/foo.cf