Better Error Handling and Logging

Summary

Better error handling (less exceptions, more error codes) and logging (for more detailed error handling and configuration issues).

Issue: https://gitlab.indiscale.com/caosdb/customers/lfpb/management/-/issues/390

See also the tests in caosdb-cppinttest!2 (merged)

Focus

• The TransactionStatus class represents the current status of a Transaction instance.
• The TransactionCode is a numeric representation of the status and intended for c-compatible error handling.
• Messages and Message classes handle the errors, warnings and info messages that come from the server.
• Logging: The logging can be configured via a json file. Otherwise the default configuration is in place.
• A lot of code was moved in order to comply with -pedantic flag. Don't waste your time

Test Environment

Unit tests should run as described in README_SETUP

Integration tests should run against a linkahead in the f-grpc branch

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related Issues
• Up-to-date CHANGELOG.md
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there spezifications? Are they satisfied?

For further good practices have a look at our review guidelines.

src/caosdb/configuration.cpp

@@ -19,17 +19,29 @@
  *
  */
 #include "caosdb/configuration.h"
-#include <cstdlib>                            // for getenv
-#include <cassert>                            // for assert
-#include <string>                             // for char_traits, string
-#include "boost/iterator/iterator_facade.hpp" // for iterator_facade_base
-#include "boost/json/impl/object.hpp"         // for object::at, object::begin
-#include "boost/json/string.hpp"              // for string
-#include "boost/json/string_view.hpp"         // for string_view
-#include "caosdb/authentication.h"            // for PlainPasswordAuthentic...
-#include "caosdb/connection.h"                // for TlsConnectionConfiguration
-#include "caosdb/constants.h"                 // for LIBCAOSDB_CONFIGURATIO...
-#include "caosdb/exceptions.h"                // for ConfigurationError
+#include "boost/iterator/iterator_facade.hpp"         // for iterator_facad...
+#include "boost/json/impl/object.hpp"                 // for object::at
+#include "boost/json/string.hpp"                      // for string
+#include "boost/json/string_view.hpp"                 // for string_view
+#include "boost/log/core/record.hpp"                  // for record
+#include "boost/log/sources/record_ostream.hpp"       // for basic_record_o...
+#include "boost/preprocessor/seq/limits/enum_256.hpp" // for BOOST_PP_SEQ_E...
+#include "boost/preprocessor/seq/limits/size_256.hpp" // for BOOST_PP_SEQ_S...
+#include "caosdb/authentication.h"                    // for Authenticator
+#include "caosdb/connection.h"                        // for ConnectionManager
+#include "caosdb/constants.h"                         // for LIBCAOSDB_CONF...
+#include "caosdb/exceptions.h"                        // for ConfigurationE...
+#include "caosdb/log_level.h"                         // for CAOSDB_DEFAULT...
+#include "caosdb/status_code.h"                       // for StatusCode
+#include "caosdb/utility.h"                           // for get_home_direc...
+#include <bits/exception.h>                           // for exception
+#include <cassert>                                    // for assert
+#include <cstdlib>                                    // for getenv
+#include <grpcpp/security/credentials.h>              // for SslCredentials
+#include <iterator>                                   // for next
+#include <map>                                        // for map
+#include <stdexcept>                                  // for out_of_range
+#include <string>                                     // for string, operator+
 
 namespace caosdb::configuration {
 using boost::filesystem::exists;
@@ -38,14 +50,118 @@ using boost::json::object;
 using boost::json::value;
 using caosdb::authentication::Authenticator;
 using caosdb::authentication::PlainPasswordAuthenticator;
-using caosdb::connection::CertificateProvider;
-using caosdb::connection::ConnectionConfiguration;
-using caosdb::connection::InsecureConnectionConfiguration;
-using caosdb::connection::PemFileCertificateProvider;
-using caosdb::connection::TlsConnectionConfiguration;
+using caosdb::connection::ConnectionManager;
 using caosdb::exceptions::ConfigurationError;
+using caosdb::logging::ConsoleSinkConfiguration;
+using caosdb::logging::FileSinkConfiguration;
+using caosdb::logging::LoggingConfiguration;
+using caosdb::logging::SinkConfiguration;
+using caosdb::logging::SyslogSinkConfiguration;
 using caosdb::utility::get_home_directory;
 using caosdb::utility::load_json_file;
+using caosdb::utility::load_string_file;
+using grpc::InsecureChannelCredentials;
+using grpc::SslCredentials;
+using grpc::SslCredentialsOptions;
+
+PemFileCertificateProvider::PemFileCertificateProvider(const path &path) {
+  this->certificate_provider = load_string_file(path);
+}
+
+auto PemFileCertificateProvider::GetCertificatePem() const -> std::string {
+  return this->certificate_provider;
+}
+
+PemCertificateProvider::PemCertificateProvider(
+  const std::string &certificate_provider) {
+  this->certificate_provider = certificate_provider;
+}
+
+auto PemCertificateProvider::GetCertificatePem() const -> std::string {
+  return this->certificate_provider;
+}
+
+ConnectionConfiguration::ConnectionConfiguration(const std::string &host,
+                                                 int port) {
+  this->host = host;
+  this->port = port;
+}
+
+auto ConnectionConfiguration::GetHost() const -> std::string {
+  return this->host;
+}
+
+auto ConnectionConfiguration::GetPort() const -> int { return this->port; }
+
+auto operator<<(std::ostream &out, const ConnectionConfiguration &configuration)
+  -> std::ostream & {
+  out << configuration.ToString();
+  return out;
+}
+
+InsecureConnectionConfiguration::InsecureConnectionConfiguration(
+  const std::string &host, int port)
+  : ConnectionConfiguration(host, port) {
+  this->credentials = InsecureChannelCredentials();
+}
+
+auto InsecureConnectionConfiguration::GetChannelCredentials() const
+  -> std::shared_ptr<ChannelCredentials> {
+  return this->credentials;
+}
+
+auto InsecureConnectionConfiguration::ToString() const -> std::string {
+  return "InsecureConnectionConfiguration(" + this->GetHost() + "," +
+         std::to_string(this->GetPort()) + ")";
+}
+
+TlsConnectionConfiguration::TlsConnectionConfiguration(const std::string &host,
+                                                       int port)
+  : ConnectionConfiguration(host, port) {
+  SslCredentialsOptions options;
+  this->credentials = SslCredentials(options);
+}
+
+TlsConnectionConfiguration::TlsConnectionConfiguration(
+  const std::string &host, int port,
+  const CertificateProvider &certificate_provider)
+  : ConnectionConfiguration(host, port) {
+  SslCredentialsOptions options;
+  options.pem_root_certs = certificate_provider.GetCertificatePem();
+  this->credentials = SslCredentials(options);
+}
+
+TlsConnectionConfiguration::TlsConnectionConfiguration(
+  const std::string &host, int port, const Authenticator &authenticator)
+  : ConnectionConfiguration(host, port) {
+
+  SslCredentialsOptions options;
+  this->credentials = grpc::CompositeChannelCredentials(
+    SslCredentials(options), authenticator.GetCallCredentials());
+}
+
+TlsConnectionConfiguration::TlsConnectionConfiguration(
+  const std::string &host, int port,
+  const CertificateProvider &certificate_provider,
+  const Authenticator &authenticator)
+  : ConnectionConfiguration(host, port) {
+
+  SslCredentialsOptions options;
+  options.pem_root_certs = certificate_provider.GetCertificatePem();
+  this->credentials = grpc::CompositeChannelCredentials(
+    SslCredentials(options), authenticator.GetCallCredentials());
+}
+
+auto TlsConnectionConfiguration::GetChannelCredentials() const
+  -> std::shared_ptr<ChannelCredentials> {
+  return this->credentials;
+}
+
+auto TlsConnectionConfiguration::ToString() const -> std::string {
+  return "TlsConnectionConfiguration(" + this->GetHost() + "," +
+         std::to_string(this->GetPort()) + "," + this->certificate_provider +
+         ")";
+}
 
 auto ConnectionConfigurationHelper::CreateCertificateProvider(
   const object &from) const -> std::unique_ptr<CertificateProvider> {
@@ -53,11 +169,17 @@ auto ConnectionConfigurationHelper::CreateCertificateProvider(
   if (from.contains("server_certificate_path")) {
     const value &path_str = from.at("server_certificate_path");
     assert(path_str.is_string() == true);
+    const path certificate_file = path(path_str.as_string().c_str());
+    if (!exists(certificate_file)) {
+      throw ConfigurationError(
+        "File does not exist (server_certificate_path): " +
+        certificate_file.string());
+    }
     certificate_provider = std::make_unique<PemFileCertificateProvider>(
-      std::string(path_str.as_string().c_str()));
+      path(path_str.as_string().c_str()));
   }
   return certificate_provider;
-};
+}
 
 auto ConnectionConfigurationHelper::CreateAuthenticator(
   const object &from) const -> std::unique_ptr<Authenticator> {
@@ -87,7 +209,7 @@ auto ConnectionConfigurationHelper::CreateAuthenticator(
     }
   }
   return authenticator;
-};
+}
 
 auto ConnectionConfigurationHelper::CreateConnectionConfiguration(
   const bool tls, const std::string &host, const int port,
@@ -113,7 +235,7 @@ auto ConnectionConfigurationHelper::CreateConnectionConfiguration(
   } else {
     return std::make_unique<InsecureConnectionConfiguration>(host, port);
   }
-};
+}
 
 auto ConnectionConfigurationHelper::IsTls(const object &from) const -> bool {
   bool tls = true;
@@ -123,7 +245,7 @@ auto ConnectionConfigurationHelper::IsTls(const object &from) const -> bool {
     tls = tls_switch.as_bool();
   }
   return tls;
-};
+}
 
 auto ConnectionConfigurationHelper::CreateConnectionConfiguration(
   const object &from) const -> std::unique_ptr<ConnectionConfiguration> {
@@ -145,16 +267,119 @@ auto ConnectionConfigurationHelper::CreateConnectionConfiguration(
     tls, std::string(host.as_string().c_str()),
     static_cast<int>(port.as_int64()), certificate_provider.get(),
     authenticator.get());
-};
+}
+
+auto LoggingConfigurationHelper::CreateConsoleSinkConfiguration(
+  const object & /*from*/, const std::string &name, int level) const
+  -> std::shared_ptr<caosdb::logging::SinkConfiguration> {
+  auto result = std::make_shared<ConsoleSinkConfiguration>(name, level);
+  return result;
+}
+
+auto LoggingConfigurationHelper::CreateSyslogSinkConfiguration(
+  const object & /*from*/, const std::string &name, int level) const
+  -> std::shared_ptr<caosdb::logging::SinkConfiguration> {
+  auto result = std::make_shared<SyslogSinkConfiguration>(name, level);
+  return result;
+}
+
+auto LoggingConfigurationHelper::CreateFileSinkConfiguration(
+  const object &from, const std::string &name, int level) const
+  -> std::shared_ptr<caosdb::logging::SinkConfiguration> {
+  auto result = std::make_shared<FileSinkConfiguration>(name, level);
+  if (from.contains("directory")) {
+    result->SetDirectory(from.at("directory").as_string().c_str());
+  }
+  return result;
+}
+
+auto LoggingConfigurationHelper::CreateSinkConfiguration(
+  const object &from, const std::string &name, int default_level) const
+  -> std::shared_ptr<caosdb::logging::SinkConfiguration> {
+  assert(from.contains("destination"));
+  const auto &destination =
+    std::string(from.at("destination").as_string().c_str());
+  int level = from.contains("level")
+                ? static_cast<int>(from.at("level").as_int64())
+                : default_level;
+
+  if (destination == "file") {
+    return CreateFileSinkConfiguration(from, name, level);
+  } else if (destination == "console") {
+    return CreateConsoleSinkConfiguration(from, name, level);
+  } else if (destination == "syslog") {
+    return CreateSyslogSinkConfiguration(from, name, level);
+  } else {
+    throw ConfigurationError("Unknown sink destination: " + destination);
+  }
+}
+
+auto LoggingConfigurationHelper::CreateLoggingConfiguration(
+  const object &from) const -> LoggingConfiguration {
+  auto default_level_str = from.contains("level")
+                             ? std::string(from.at("level").as_string().c_str())
+                             : "";
+  int default_level = 0;
+  static std::map<std::string, int> log_level_names = {
+    {"", CAOSDB_DEFAULT_LOG_LEVEL},    {"off", CAOSDB_LOG_LEVEL_OFF},
+    {"fatal", CAOSDB_LOG_LEVEL_FATAL}, {"error", CAOSDB_LOG_LEVEL_ERROR},
+    {"warn", CAOSDB_LOG_LEVEL_WARN},   {"info", CAOSDB_LOG_LEVEL_INFO},
+    {"debug", CAOSDB_LOG_LEVEL_DEBUG}, {"trace", CAOSDB_LOG_LEVEL_TRACE},
+    {"all", CAOSDB_LOG_LEVEL_ALL}};
+
+  try {
+    default_level = CAOSDB_DEFAULT_LOG_LEVEL;
+  } catch (const std::out_of_range &exc) {
+    throw ConfigurationError("Unknown log level: " + default_level_str);
+  }
+
+  auto result = LoggingConfiguration(default_level);
+  if (default_level == CAOSDB_LOG_LEVEL_OFF) {
+    return result;
+  }
 
-auto ConfigurationManager::mReset() -> void {
-  mClear();
-  InitializeDefaults();
-};
+  const auto &sinks = from.at("sinks").as_object();
+  if (!sinks.empty()) {
+    const auto *elem = sinks.begin();
 
-auto ConfigurationManager::mClear() -> void {
-  json_configuration = value(nullptr);
-  ConnectionManager::Reset();
+    while (elem != sinks.end()) {
+      result.AddSink(CreateSinkConfiguration(
+        elem->value().as_object(), elem->key().to_string(), default_level));
+      elem = std::next(elem);
+    }
+  }
+
+  return result;
+}
+
+auto ConfigurationManager::mReset() noexcept -> int {
+  try {
+    mClear();
+    InitializeDefaults();
+    return StatusCode::SUCCESS;
+  } catch (const caosdb::exceptions::Exception &exc) {
+    return exc.GetCode();
+  } catch (const std::exception &exc) {
+    CAOSDB_LOG_ERROR(logger_name)
+      << "Unknown error during the reset of the ConfigurationManager: "
+      << exc.what();
+    return StatusCode::CONFIGURATION_ERROR;
+  }
+}
+
+auto ConfigurationManager::mClear() noexcept -> int {
+  try {
+    json_configuration = value(nullptr);
+    ConnectionManager::Reset();
+    return StatusCode::SUCCESS;
+  } catch (const caosdb::exceptions::Exception &exc) {
+    return exc.GetCode();
+  } catch (const std::exception &exc) {
+    CAOSDB_LOG_ERROR(logger_name)
+      << "Unknown error during the reset of the ConfigurationManager: "
+      << exc.what();
+    return StatusCode::CONFIGURATION_ERROR;
+  }
 }
 
 auto ConfigurationManager::mLoadSingleJSONConfiguration(const path &json_file)
@@ -167,15 +392,14 @@ auto ConfigurationManager::mLoadSingleJSONConfiguration(const path &json_file)
   }
 
   json_configuration = load_json_file(json_file);
-  // TODO(far future) validate against json-schema
-};
+}
 
 auto ConfigurationManager::mGetConnectionConfiguration(
   const std::string &name) const -> std::unique_ptr<ConnectionConfiguration> {
   auto connection_json = GetConnection(name);
   return connection_configuration_helper.CreateConnectionConfiguration(
     connection_json);
-};
+}
 
 auto ConfigurationManager::mGetDefaultConnectionName() const -> std::string {
   auto connections = GetConnections();
@@ -196,7 +420,7 @@ auto ConfigurationManager::mGetDefaultConnectionName() const -> std::string {
     return connections.begin()->key().to_string();
   }
   throw ConfigurationError("Could not determine the default connection.");
-};
+}
 
 auto ConfigurationManager::GetConfiguration() const -> const object & {
   if (json_configuration.is_null()) {
@@ -204,7 +428,7 @@ auto ConfigurationManager::GetConfiguration() const -> const object & {
   }
   assert(json_configuration.is_object());
   return json_configuration.as_object();
-};
+}
 
 auto ConfigurationManager::GetConnections() const -> const object & {
   const auto &configuration = GetConfiguration();
@@ -224,7 +448,7 @@ auto ConfigurationManager::GetConnections() const -> const object & {
       "This CaosDB client hasn't any configured connections.");
   }
   return connections_object;
-};
+}
 
 auto ConfigurationManager::GetConnection(const std::string &name) const
   -> const object & {
@@ -236,9 +460,9 @@ auto ConfigurationManager::GetConnection(const std::string &name) const
   }
   throw ConfigurationError("The connection '" + name +
                            "' has not been defined.");
-};
+}
 
-auto ConfigurationManager::InitializeDefaults() -> void {
+auto ConfigurationManager::InitializeDefaults() -> int {
 
   // find the configuration file...
   std::unique_ptr<path> configuration_file_path;
@@ -282,9 +506,23 @@ auto ConfigurationManager::InitializeDefaults() -> void {
   if (configuration_file_path != nullptr) {
     // TODO(tf): log which file has been used.
     mLoadSingleJSONConfiguration(*configuration_file_path);
+  }
+
+  // Logging in the configuration leads to additional content.
+  if (this->json_configuration.is_object() &&
+      this->json_configuration.as_object().contains("logging")) {
+    LoggingConfiguration logging_configuration =
+      logging_configuration_helper.CreateLoggingConfiguration(
+        json_configuration.at("logging").as_object());
+    logging::initialize_logging(logging_configuration);
   } else {
-    // TODO(tf): log warning: no configuration files has been found
+    logging::initialize_logging_defaults();
+    CAOSDB_LOG_WARN(logger_name) << "No configuration has been found.";
   }
+
+  return 0;
 }
 
+const int IS_INITIALIZED = ConfigurationManager::Reset();
+
 } // namespace caosdb::configuration