Schema based configuration

.gitignore

@@ -53,3 +53,6 @@ calgrind.*
 *.hdf
 wct-deps.dot
 
+
+.direnv
+.envrc

apps/inc/WireCellApps/Main.h

@@ -91,11 +91,18 @@ namespace WireCell {
  /// Call to finalize any terminal components.
  void finalize();
 
+ /// Return the compiled version string
+ std::string version() const;
+
  private:
  ConfigManager m_cfgmgr;
  std::vector<std::string> m_plugins, m_apps, m_cfgfiles, m_load_path;
  Persist::externalvars_t m_extvars, m_extcode, m_tlavars, m_tlacode;
  Log::logptr_t l;
+
+ // Limit number of threads. 0 means set no limit. This is
+ // only relevant if we are built with TBB support
+ int m_threads{0};
  };
 
 } // namespace WireCell

apps/src/Main.cxx

@@ -3,13 +3,15 @@
 
 #include "WireCellApps/Main.h"
 
+#include "WireCellUtil/Version.h"
 #include "WireCellUtil/NamedFactory.h"
 #include "WireCellUtil/String.h"
 #include "WireCellUtil/Point.h"
 
 #include "WireCellIface/IConfigurable.h"
 #include "WireCellIface/ITerminal.h"
 #include "WireCellIface/IApplication.h"
+#include "WireCellIface/INamed.h"
 
 #include <boost/program_options.hpp>
 #include <boost/algorithm/string.hpp>
@@ -25,6 +27,10 @@
 #include <fftw3.h>
 #endif
 
+#if HAVE_TBB_LIB
+#include <tbb/global_control.h>
+#endif
+
 using namespace WireCell;
 using namespace std;
 namespace po = boost::program_options;
@@ -79,7 +85,16 @@ int Main::cmdline(int argc, char* argv[])
  "specify a Jsonnet top level arguments variable=<code>")
 
  ("path,P", po::value<vector<string> >(),
- "add to JSON/Jsonnet search path");
+ "add to JSON/Jsonnet search path")
+#if HAVE_TBB_LIB
+ ("threads,t", po::value<int>(),
+ "limit number of threads used")
+#endif
+
+ ("version,v", 
+ "print the compiled version to stdout")
+
+ ;
  // clang-format on
 
  po::variables_map opts;
@@ -171,13 +186,28 @@ int Main::cmdline(int argc, char* argv[])
  }
  }
 
+#ifdef HAVE_TBB_LIB
+ if (opts.count("threads")) {
+ m_threads = opts["threads"].as<int>();
+ }
+#endif
+
+ if (opts.count("version")) {
+ std::cout << version() << std::endl;
+ }
+
  // Maybe make this cmdline configurable. For now, set all
  // backends the same.
  Log::set_pattern("[%H:%M:%S.%03e] %L [%^%=8n%$] %v");
 
  return 0;
 }
 
+std::string Main::version() const
+{
+ return WireCell::version;
+}
+
 void Main::add_plugin(const std::string& libname) { m_plugins.push_back(libname); }
 
 void Main::add_app(const std::string& tn) { m_apps.push_back(tn); }
@@ -209,6 +239,9 @@ void Main::add_path(const std::string& dirname) { m_load_path.push_back(dirname)
 
 void Main::initialize()
 {
+ // Here we got thought the boot-up sequence steps.
+
+ // Load configuration files
  for (auto filename : m_cfgfiles) {
  l->info("loading config file {} ...", filename);
  Persist::Parser p(m_load_path, m_extvars, m_extcode, m_tlavars, m_tlacode);
@@ -217,7 +250,7 @@ void Main::initialize()
  l->info("...done");
  }
 
- // Find if we have our special configuration entry
+ // Find if we have our own special configuration entry
  int ind = m_cfgmgr.index("wire-cell");
  Configuration main_cfg = m_cfgmgr.pop(ind);
  if (!main_cfg.isNull()) {
@@ -243,11 +276,9 @@ void Main::initialize()
  pm.add(pname, lname);
  }
 
- // Apply any user configuration. This is a two step. First, just
- // assure all the components referenced in the configuration
- // sequence can be instantiated. Then, find them again and
- // actually configure them. This way, any problems fails fast.
+ // Apply any component configuration sequence.
 
+ // Instantiation
  for (auto c : m_cfgmgr.all()) {
  if (c.isNull()) {
  continue; // allow and ignore any totally empty configurations
@@ -265,6 +296,23 @@ void Main::initialize()
  l->info("constructing app: \"{}\"", c);
  Factory::lookup_tn<IApplication>(c);
  }
+
+ // Give any named components their name.
+ for (auto c : m_cfgmgr.all()) {
+ if (c.isNull()) {
+ continue; // allow and ignore any totally empty configurations
+ }
+ string type = get<string>(c, "type");
+ string name = get<string>(c, "name");
+ auto namobj = Factory::find_maybe<INamed>(type, name); // doesn't throw.
+ if (!namobj) {
+ continue;
+ }
+ namobj->set_name(name);
+ }
+
+ // Finally, ask any configurables for their default, merge with
+ // user config and give back.
  for (auto c : m_cfgmgr.all()) {
  if (c.isNull()) {
  continue; // allow and ignore any totally empty configurations
@@ -295,12 +343,28 @@ void Main::operator()()
  auto a = Factory::find<IApplication>(type, name); // throws
  app_objs.push_back(a);
  }
+ l->debug("executing {} apps, thread limit {}:",
+ m_apps.size(), m_threads);
+
+#if HAVE_TBB_LIB
+ std::unique_ptr<tbb::global_control> gc;
+ if (m_threads) {
+ gc = std::make_unique<tbb::global_control>(
+ tbb::global_control::max_allowed_parallelism,
+ m_threads);
+ }
+ l->debug("executing {} apps, thread limit {}:",
+ m_apps.size(), m_threads);
+#else
  l->debug("executing {} apps:", m_apps.size());
+#endif
+
  for (size_t ind = 0; ind < m_apps.size(); ++ind) {
  auto aobj = app_objs[ind];
  l->debug("executing app: \"{}\"", m_apps[ind]);
  aobj->execute(); // throws
  }
+
 }
 
 void Main::finalize()

apps/wscript_build

@@ -1,3 +1,8 @@
-bld.smplpkg('WireCellApps', use='WireCellIface FFTWTHREADS')
+if 'HAVE_TBB' in bld.env:
+ bld.smplpkg('WireCellApps', use='WireCellIface FFTWTHREADS TBB')
+else:
+ bld.smplpkg('WireCellApps', use='WireCellIface FFTWTHREADS')
+
+
 # app_use='WireCellUtil JSONNET',
 # test_use='WireCellUtil DYNAMO BOOST JSONCPP JSONNET')

aux/inc/WireCellAux/ClusterHelpers.h

@@ -0,0 +1,33 @@
+// Provide some helper functions for working with ICluster
+
+#ifndef WIRECELLAUX_CLUSTERJSONIFY
+#define WIRECELLAUX_CLUSTERJSONIFY
+
+#include "WireCellUtil/Configuration.h"
+#include "WireCellIface/IFrame.h"
+#include "WireCellIface/ISlice.h"
+#include "WireCellIface/ICluster.h"
+
+#include <string>
+
+namespace WireCell::Aux {
+ /// Return JSON representation of the cluster.
+ Json::Value jsonify(const ICluster::pointer& cluster,
+ double drift_speed);
+
+ /// Return name for the cluster in a canonical form suitable for
+ /// use as a file name. It only includes info related directly to
+ /// the cluster (the ident).
+ std::string name(const ICluster::pointer& cluster);
+
+ /// Return the slices in the cluster.
+ ISlice::vector find_slices(const ICluster::pointer& cluster);
+
+ /// Return the frame of the first slice in the cluster. Note, in
+ /// principle, clusters can span frames.
+ IFrame::pointer find_frame(const ICluster::pointer& cluster);
+
+}
+
+#endif
+

aux/inc/WireCellAux/Configurable.h

@@ -0,0 +1,64 @@
+/** A base class providing auto-configuration.
+
+ This works in conjunction with a config struct which is produced
+ via shema codegen.
+
+ To use
+
+ 1) Inherit your MyConfigurable class from
+ Configurable<Cfg::MyConfigurable> instead of directly
+ IConfigurable.
+
+ 2) If migrating from old, remove configure() and
+ default_configure() methods.
+
+ 3) If you configure() method held non-configuration related
+ initialization, it may move into configured().
+
+ 4) Use the protected m_cfg object to access the configuration
+ struct including any "service" type components "used" by yours.
+
+ */
+
+
+#ifndef WIRECELLAUX_CONFIGURABLE
+#define WIRECELLAUX_CONFIGURABLE
+
+#include "WireCellUtil/nljs2jcpp.hpp" // remove when ditch JsonCPP
+
+#include "WireCellIface/IConfigurable.h"
+
+namespace WireCell::Aux {
+
+ template<typename CfgStruct>
+ class Configurable : public IConfigurable {
+ public:
+ using cfg_type = CfgStruct;
+
+ // Subclass may implement in order to get post-config entry.
+ virtual void configured() { }
+
+ // Subclass need not but MAY overide IF it also calls.
+ virtual void configure(const WireCell::Configuration& jcfg)
+ {
+ nljs_t nljs = jcfg;
+ m_cfg = nljs.get<cfg_type>();
+ configured();
+ }
+
+ // Subclass need not but MAY overide IF it also calls.
+ virtual WireCell::Configuration default_configuration() const
+ {
+ const nljs_t nljs = m_cfg;
+ return nljs.get<Json::Value>();
+ }
+
+ protected:
+ // Subclass should use m_cfg directly.
+ cfg_type m_cfg;
+ };
+
+}
+
+
+#endif

aux/inc/WireCellAux/DepoTools.h

@@ -0,0 +1,29 @@
+/**
+ Some helper functions operating on IDepo
+ */
+#ifndef WIRECELLAUX_DEPOTOOLS
+#define WIRECELLAUX_DEPOTOOLS
+
+#include "WireCellIface/IDepo.h"
+#include "WireCellUtil/Array.h"
+
+namespace WireCell::Aux {
+
+ /** Fill data and info arrays from IDepoSet.
+
+ The "data" array will be shaped (ndepos,7) and each
+ 7-tuple holds: (time, charge, x, y, z, long, tran).
+
+ The "info" array is integer and shaped (nedpos, 4). Each
+ 4-tuple holds: (id, pdg, gen, child).
+
+ If "priors" is true, then run the depos through flatten() in
+ order to save them and give non-zero gen and child. O.w.,
+ only those depos in pass directly in the vector are saved.
+ */
+ void fill(Array::array_xxf& data, Array::array_xxi& info, 
+ const IDepo::vector& depos, bool priors = true);
+
+}
+
+#endif

aux/inc/WireCellAux/FrameTools.h

@@ -6,12 +6,19 @@
 
 #include "WireCellIface/IFrame.h"
 #include "WireCellUtil/Array.h"
+#include "WireCellUtil/Logging.h"
 
 namespace WireCell {
- namespace aux {
+ namespace Aux {
 
- // Print some info to cerr about frame
- void dump_frame(WireCell::IFrame::pointer frame);
+ /// Return a canonical name for the frame.
+ std::string name(const WireCell::IFrame::pointer& frame);
+
+ /// Print some info to cerr about frame. If log is null, use stderr.
+ void dump_frame(WireCell::IFrame::pointer frame, Log::logptr_t log = nullptr);
+
+ /// Return brief info string summarizing frame and trace tag.
+ std::string taginfo(const WireCell::IFrame::pointer& frame);
 
  /// Return a vector of traces which have no trace tags. Here,
  /// any frame tags are ignored. Returned vector of traces has
@@ -106,7 +113,7 @@ namespace WireCell {
  /// outside of any individual trace.
  IFrame::pointer sum(std::vector<IFrame::pointer> frames, int ident);
 
- } // namespace aux
+ } // namespace Aux
 } // namespace WireCell
 
 #endif