add clearcut upload plugin from openmhz uploader

4 files changed, 385 insertions, 1 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 88cc9aad..4de7bbb0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -324,6 +324,8 @@ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/trunk-recorder/" # source directo
 
 add_subdirectory(plugins/openmhz_uploader)
 
+add_subdirectory(plugins/clearcut_uploader)
+
 add_subdirectory(plugins/stat_socket)
 
 add_subdirectory(plugins/broadcastify_uploader)
diff --git a/plugins/clearcut_uploader/CMakeLists.txt b/plugins/clearcut_uploader/CMakeLists.txt
new file mode 100644
index 00000000..372820d4
--- /dev/null
+++ b/plugins/clearcut_uploader/CMakeLists.txt
@@ -0,0 +1,20 @@
+add_library(clearcut_uploader
+  MODULE
+    clearcut_uploader.cc
+  )
+
+target_link_libraries(clearcut_uploader trunk_recorder_library ssl crypto ${CURL_LIBRARIES} ${Boost_LIBRARIES} ${GNURADIO_PMT_LIBRARIES} ${GNURADIO_RUNTIME_LIBRARIES} ${GNURADIO_FILTER_LIBRARIES} ${GNURADIO_DIGITAL_LIBRARIES} ${GNURADIO_ANALOG_LIBRARIES} ${GNURADIO_AUDIO_LIBRARIES} ${GNURADIO_UHD_LIBRARIES} ${UHD_LIBRARIES} ${GNURADIO_BLOCKS_LIBRARIES} ${GNURADIO_OSMOSDR_LIBRARIES}  ${LIBOP25_REPEATER_LIBRARIES} gnuradio-op25_repeater ) # gRPC::grpc++_reflection protobuf::libprotobuf)
+
+if(NOT Gnuradio_VERSION VERSION_LESS "3.8")
+
+    target_link_libraries(clearcut_uploader
+    gnuradio::gnuradio-analog
+    gnuradio::gnuradio-blocks
+    gnuradio::gnuradio-digital
+    gnuradio::gnuradio-filter
+    gnuradio::gnuradio-pmt
+    )  
+
+endif()
+
+install(TARGETS clearcut_uploader LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/trunk-recorder)
diff --git a/plugins/clearcut_uploader/clearcut_uploader.cc b/plugins/clearcut_uploader/clearcut_uploader.cc
new file mode 100644
index 00000000..158ffbb8
--- /dev/null
+++ b/plugins/clearcut_uploader/clearcut_uploader.cc
@@ -0,0 +1,361 @@
+#include <curl/curl.h>
+#include <time.h>
+#include <iomanip>
+#include <vector>
+
+#include "../../trunk-recorder/call_concluder/call_concluder.h"
+#include "../../trunk-recorder/plugin_manager/plugin_api.h"
+#include <boost/dll/alias.hpp> // for BOOST_DLL_ALIAS
+#include <boost/foreach.hpp>
+#include "../trunk-recorder/gr_blocks/decoder_wrapper.h"
+#include <sys/stat.h>
+
+struct Clearcut_System_Key {
+  std::string api_key;
+  std::string short_name;
+};
+
+struct Clearcut_Uploader_Data {
+  std::vector<Clearcut_System_Key> keys;
+  std::string clearcut_server;
+};
+
+class Clearcut_Uploader : public Plugin_Api {
+  //float aggr_;
+  //my_plugin_aggregator() : aggr_(0) {}
+  Clearcut_Uploader_Data data;
+
+public:
+  std::string get_api_key(std::string short_name) {
+    for (std::vector<Clearcut_System_Key>::iterator it = data.keys.begin(); it != data.keys.end(); ++it) {
+      Clearcut_System_Key key = *it;
+      if (key.short_name == short_name) {
+        return key.api_key;
+      }
+    }
+    return "";
+  }
+  static size_t write_callback(void *contents, size_t size, size_t nmemb, void *userp) {
+    ((std::string *)userp)->append((char *)contents, size * nmemb);
+    return size * nmemb;
+  }
+  int upload(Call_Data_t call_info) {
+
+    std::string api_key = get_api_key(call_info.short_name);
+    if (api_key.size() == 0) {
+      //BOOST_LOG_TRIVIAL(error) << "[" << call_info.short_name << "]\tTG: " << talkgroup_display << "\t " << std::put_time(std::localtime(&start_time), "%c %Z") << "\tClearCut Upload failed, API Key not found in config for shortName";
+      return 0;
+    }
+    
+    std::ostringstream freq;
+    std::string freq_string;
+    freq << std::fixed << std::setprecision(0);
+    freq << call_info.freq;
+
+    std::ostringstream call_length;
+    std::string call_length_string;
+    call_length << std::fixed << std::setprecision(0);
+    call_length << call_info.length;
+
+    std::ostringstream source_list;
+    std::string source_list_string;
+    source_list << std::fixed << std::setprecision(2);
+    source_list << "[";
+
+    if (call_info.transmission_source_list.size() != 0) {
+      for (int i = 0; i < call_info.transmission_source_list.size(); i++) {
+        source_list << "{\"src\": " << std::fixed << call_info.transmission_source_list[i].source << ", \"time\": " << call_info.transmission_source_list[i].time << ", \"pos\": " << std::fixed << std::setprecision(2) << call_info.transmission_source_list[i].position << ", \"emergency\": " << call_info.transmission_source_list[i].emergency << ", \"signal_system\": \"" << call_info.transmission_source_list[i].signal_system << "\", \"tag\": \"" << call_info.transmission_source_list[i].tag << "\"}";
+        if (i < (call_info.transmission_source_list.size() - 1)) {
+          source_list << ", ";
+        } else {
+          source_list << "]";
+        }
+      }
+    } else {
+      source_list << "]";
+    }
+
+
+    std::ostringstream freq_list;
+    std::string freq_list_string;
+    freq_list << std::fixed << std::setprecision(2);
+    freq_list << "[";
+
+    if (call_info.transmission_error_list.size() != 0) {
+      for (std::size_t i = 0; i < call_info.transmission_error_list.size(); i++) {
+          freq_list << "{\"freq\": " << std::fixed << std::setprecision(0) << call_info.freq << ", \"time\": " << call_info.transmission_error_list[i].time << ", \"pos\": " << std::fixed << std::setprecision(2) << call_info.transmission_error_list[i].position << ", \"len\": " << call_info.transmission_error_list[i].total_len  << ", \"error_count\": \"" << std::setprecision(0) <<call_info.transmission_error_list[i].error_count << "\", \"spike_count\": \"" << call_info.transmission_error_list[i].spike_count << "\"}"; 
+  
+        if (i < (call_info.transmission_error_list.size() - 1)) {
+          freq_list << ", ";
+        } else {
+          freq_list << "]";
+        }
+      }
+    }else {
+      freq_list << "]";
+    }
+
+
+
+    char formattedTalkgroup[62];
+    snprintf(formattedTalkgroup, 61, "%c[%dm%10ld%c[0m", 0x1B, 35, call_info.talkgroup, 0x1B);
+    std::string talkgroup_display = boost::lexical_cast<std::string>(formattedTalkgroup);
+    CURL *curl;
+    CURLMcode res;
+    CURLM *multi_handle;
+    int still_running = 0;
+    std::string response_buffer;
+    freq_string = freq.str();
+
+    source_list_string = source_list.str();
+    freq_list_string = freq_list.str();
+    call_length_string = call_length.str();
+
+    struct curl_httppost *formpost = NULL;
+    struct curl_httppost *lastptr = NULL;
+    struct curl_slist *headerlist = NULL;
+
+    /* Fill in the file upload field. This makes libcurl load data from
+     the given file name when curl_easy_perform() is called. */
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "call",
+                 CURLFORM_FILE, call_info.converted,
+                 CURLFORM_CONTENTTYPE, "application/octet-stream",
+                 CURLFORM_END);
+
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "freq",
+                 CURLFORM_COPYCONTENTS, freq_string.c_str(),
+                 CURLFORM_END);
+
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "start_time",
+                 CURLFORM_COPYCONTENTS, boost::lexical_cast<std::string>(call_info.start_time).c_str(),
+                 CURLFORM_END);
+
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "stop_time",
+                 CURLFORM_COPYCONTENTS, boost::lexical_cast<std::string>(call_info.stop_time).c_str(),
+                 CURLFORM_END);
+
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "call_length",
+                 CURLFORM_COPYCONTENTS, call_length_string.c_str(),
+                 CURLFORM_END);
+
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "talkgroup_num",
+                 CURLFORM_COPYCONTENTS, boost::lexical_cast<std::string>(call_info.talkgroup).c_str(),
+                 CURLFORM_END);
+
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "emergency",
+                 CURLFORM_COPYCONTENTS, boost::lexical_cast<std::string>(call_info.emergency).c_str(),
+                 CURLFORM_END);
+
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "api_key",
+                 CURLFORM_COPYCONTENTS, api_key.c_str(),
+                 CURLFORM_END);
+
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "source_list",
+                 CURLFORM_COPYCONTENTS, source_list_string.c_str(),
+                 CURLFORM_END);
+    curl_formadd(&formpost,
+                 &lastptr,
+                 CURLFORM_COPYNAME, "freq_list",
+                 CURLFORM_COPYCONTENTS, freq_list_string.c_str(),
+                 CURLFORM_END);
+
+    curl = curl_easy_init();
+    multi_handle = curl_multi_init();
+
+    /* initialize custom header list (stating that Expect: 100-continue is not wanted */
+    headerlist = curl_slist_append(headerlist, "Expect:");
+    if (curl && multi_handle) {
+      std::string url = data.clearcut_server + "/api/v1/upload";
+
+      /* what URL that receives this POST */
+      curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+
+      curl_easy_setopt(curl, CURLOPT_USERAGENT, "TrunkRecorder1.0");
+      curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist);
+      curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
+
+      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+      curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_buffer);
+
+      curl_multi_add_handle(multi_handle, curl);
+
+      curl_multi_perform(multi_handle, &still_running);
+
+      while (still_running) {
+        struct timeval timeout;
+        int rc;       /* select() return code */
+        CURLMcode mc; /* curl_multi_fdset() return code */
+
+        fd_set fdread;
+        fd_set fdwrite;
+        fd_set fdexcep;
+        int maxfd = -1;
+
+        long curl_timeo = -1;
+
+        FD_ZERO(&fdread);
+        FD_ZERO(&fdwrite);
+        FD_ZERO(&fdexcep);
+
+        /* set a suitable timeout to play around with */
+        timeout.tv_sec = 1;
+        timeout.tv_usec = 0;
+
+        curl_multi_timeout(multi_handle, &curl_timeo);
+        if (curl_timeo >= 0) {
+          timeout.tv_sec = curl_timeo / 1000;
+          if (timeout.tv_sec > 1)
+            timeout.tv_sec = 1;
+          else
+            timeout.tv_usec = (curl_timeo % 1000) * 1000;
+        }
+
+        /* get file descriptors from the transfers */
+        mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+
+        if (mc != CURLM_OK) {
+          fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
+          break;
+        }
+
+        /* On success the value of maxfd is guaranteed to be >= -1. We call
+         select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
+         no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
+         to sleep 100ms, which is the minimum suggested value in the
+         curl_multi_fdset() doc. */
+
+        if (maxfd == -1) {
+          /* Portable sleep for platforms other than Windows. */
+          struct timeval wait = {0, 100 * 1000}; /* 100ms */
+          rc = select(0, NULL, NULL, NULL, &wait);
+        } else {
+          /* Note that on some platforms 'timeout' may be modified by select().
+           If you need access to the original value save a copy beforehand. */
+          rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
+        }
+
+        switch (rc) {
+        case -1:
+          /* select error */
+          break;
+        case 0:
+        default:
+          /* timeout or readable/writable sockets */
+          curl_multi_perform(multi_handle, &still_running);
+          break;
+        }
+      }
+
+      long response_code;
+      curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+
+      res = curl_multi_cleanup(multi_handle);
+
+      /* always cleanup */
+      curl_easy_cleanup(curl);
+
+      /* then cleanup the formpost chain */
+      curl_formfree(formpost);
+
+      /* free slist */
+      curl_slist_free_all(headerlist);
+
+      if (res == CURLM_OK && response_code == 200) {
+        struct stat file_info;
+        stat(call_info.converted, &file_info);
+
+        BOOST_LOG_TRIVIAL(info) << "[" << call_info.short_name << "]\t\033[0;34m" << call_info.call_num << "C\033[0m\tTG: " << call_info.talkgroup_display << "\tFreq: " << format_freq(call_info.freq) << "\tClearCut Upload Success - file size: " << file_info.st_size;
+        ;
+        return 0;
+      }
+    }
+    BOOST_LOG_TRIVIAL(error) << "[" << call_info.short_name << "]\t\033[0;34m" << call_info.call_num << "C\033[0m\tTG: " << call_info.talkgroup_display << "\tFreq: " << format_freq(call_info.freq) << "\tClearCut Upload Error: " << response_buffer;
+    return 1;
+  }
+
+  int call_end(Call_Data_t call_info) {
+    return upload(call_info);
+  }
+
+  int parse_config(boost::property_tree::ptree &cfg) {
+    boost::optional<std::string> upload_server_exists = cfg.get_optional<std::string>("clearcutServer");
+    if (!upload_server_exists) {
+      return 1;
+    }
+
+    this->data.clearcut_server = cfg.get<std::string>("clearcutServer", "");
+    BOOST_LOG_TRIVIAL(info) << "ClearCut Server: " << this->data.clearcut_server;
+
+    // from: http://www.zedwood.com/article/cpp-boost-url-regex
+    boost::regex ex("(http|https)://([^/ :]+):?([^/ ]*)(/?[^ #?]*)\\x3f?([^ #]*)#?([^ ]*)");
+    boost::cmatch what;
+
+    if (!regex_match(this->data.clearcut_server.c_str(), what, ex)) {
+      BOOST_LOG_TRIVIAL(error) << "Unable to parse Server URL\n";
+      return 1;
+    } 
+    // Gets the API key for each system, if defined
+    BOOST_FOREACH (boost::property_tree::ptree::value_type &node, cfg.get_child("systems")) {
+      boost::optional<boost::property_tree::ptree &> clearcut_exists = node.second.get_child_optional("clearcutApiKey");
+      if (clearcut_exists) {
+        Clearcut_System_Key key;
+        key.api_key = node.second.get<std::string>("clearcutApiKey", "");
+        key.short_name = node.second.get<std::string>("shortName", "");
+        BOOST_LOG_TRIVIAL(info) << "Uploading calls to ClearCut for: " << key.short_name;
+        this->data.keys.push_back(key);
+      }
+    }
+
+    if (this->data.keys.size() == 0) {
+      BOOST_LOG_TRIVIAL(error) << "ClearCut Server set, but no Systems are configured\n";
+      return 1;
+    }
+
+    return 0;
+  }
+
+  /*
+ int init(Config *config, std::vector<Source *> sources, std::vector<System *> systems) { return 0; }
+   int start() { return 0; }
+   int stop() { return 0; }
+   int poll_one() { return 0; }
+   int signal(long unitId, const char *signaling_type, gr::blocks::SignalType sig_type, Call *call, System *system, Recorder *recorder) { return 0; }
+   int audio_stream(Recorder *recorder, float *samples, int sampleCount) { return 0; }
+   int call_start(Call *call) { return 0; }
+   int calls_active(std::vector<Call *> calls) { return 0; }
+   int setup_recorder(Recorder *recorder) { return 0; }
+   int setup_system(System *system) { return 0; }
+   int setup_systems(std::vector<System *> systems) { return 0; }
+   int setup_sources(std::vector<Source *> sources) { return 0; }
+   int setup_config(std::vector<Source *> sources, std::vector<System *> systems) { return 0; }
+   int system_rates(std::vector<System *> systems, float timeDiff) { return 0; }
+*/
+  // Factory method
+  static boost::shared_ptr<Clearcut_Uploader> create() {
+    return boost::shared_ptr<Clearcut_Uploader>(
+        new Clearcut_Uploader());
+  }
+};
+
+BOOST_DLL_ALIAS(
+    Clearcut_Uploader::create, // <-- this function is exported with...
+    create_plugin             // <-- ...this alias name
+)
diff --git a/trunk-recorder/main.cc b/trunk-recorder/main.cc
index 7584cb81..2ebbf205 100644
--- a/trunk-recorder/main.cc
+++ b/trunk-recorder/main.cc
@@ -570,8 +570,9 @@ bool load_config(string config_file) {
     }
 
     BOOST_LOG_TRIVIAL(info) << "\n\n-------------------------------------\nPLUGINS\n-------------------------------------\n";
-    add_internal_plugin("openmhz_uploader", "libopenmhz_uploader.so", pt);
     add_internal_plugin("broadcastify_uploader", "libbroadcastify_uploader.so", pt);
+    add_internal_plugin("openmhz_uploader", "libopenmhz_uploader.so", pt);
+    add_internal_plugin("clearcut_uploader", "libclearcut_uploader.so", pt);
     add_internal_plugin("unit_script", "libunit_script.so", pt);
     add_internal_plugin("stat_socket", "libstat_socket.so", pt);
     initialize_plugins(pt, &config, sources, systems);