From 3e59bd745e3cd56736494a942c84e9d716cdb916 Mon Sep 17 00:00:00 2001 From: Galen Guyer Date: Wed, 26 Apr 2023 12:41:28 -0400 Subject: add clearcut upload plugin from openmhz uploader --- CMakeLists.txt | 2 + plugins/clearcut_uploader/CMakeLists.txt | 20 ++ plugins/clearcut_uploader/clearcut_uploader.cc | 361 +++++++++++++++++++++++++ trunk-recorder/main.cc | 3 +- 4 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 plugins/clearcut_uploader/CMakeLists.txt create mode 100644 plugins/clearcut_uploader/clearcut_uploader.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 88cc9aad..4de7bbb0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -324,6 +324,8 @@ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/trunk-recorder/" # source directo add_subdirectory(plugins/openmhz_uploader) +add_subdirectory(plugins/clearcut_uploader) + add_subdirectory(plugins/stat_socket) add_subdirectory(plugins/broadcastify_uploader) diff --git a/plugins/clearcut_uploader/CMakeLists.txt b/plugins/clearcut_uploader/CMakeLists.txt new file mode 100644 index 00000000..372820d4 --- /dev/null +++ b/plugins/clearcut_uploader/CMakeLists.txt @@ -0,0 +1,20 @@ +add_library(clearcut_uploader + MODULE + clearcut_uploader.cc + ) + +target_link_libraries(clearcut_uploader trunk_recorder_library ssl crypto ${CURL_LIBRARIES} ${Boost_LIBRARIES} ${GNURADIO_PMT_LIBRARIES} ${GNURADIO_RUNTIME_LIBRARIES} ${GNURADIO_FILTER_LIBRARIES} ${GNURADIO_DIGITAL_LIBRARIES} ${GNURADIO_ANALOG_LIBRARIES} ${GNURADIO_AUDIO_LIBRARIES} ${GNURADIO_UHD_LIBRARIES} ${UHD_LIBRARIES} ${GNURADIO_BLOCKS_LIBRARIES} ${GNURADIO_OSMOSDR_LIBRARIES} ${LIBOP25_REPEATER_LIBRARIES} gnuradio-op25_repeater ) # gRPC::grpc++_reflection protobuf::libprotobuf) + +if(NOT Gnuradio_VERSION VERSION_LESS "3.8") + + target_link_libraries(clearcut_uploader + gnuradio::gnuradio-analog + gnuradio::gnuradio-blocks + gnuradio::gnuradio-digital + gnuradio::gnuradio-filter + gnuradio::gnuradio-pmt + ) + +endif() + +install(TARGETS clearcut_uploader LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/trunk-recorder) diff --git a/plugins/clearcut_uploader/clearcut_uploader.cc b/plugins/clearcut_uploader/clearcut_uploader.cc new file mode 100644 index 00000000..158ffbb8 --- /dev/null +++ b/plugins/clearcut_uploader/clearcut_uploader.cc @@ -0,0 +1,361 @@ +#include +#include +#include +#include + +#include "../../trunk-recorder/call_concluder/call_concluder.h" +#include "../../trunk-recorder/plugin_manager/plugin_api.h" +#include // for BOOST_DLL_ALIAS +#include +#include "../trunk-recorder/gr_blocks/decoder_wrapper.h" +#include + +struct Clearcut_System_Key { + std::string api_key; + std::string short_name; +}; + +struct Clearcut_Uploader_Data { + std::vector keys; + std::string clearcut_server; +}; + +class Clearcut_Uploader : public Plugin_Api { + //float aggr_; + //my_plugin_aggregator() : aggr_(0) {} + Clearcut_Uploader_Data data; + +public: + std::string get_api_key(std::string short_name) { + for (std::vector::iterator it = data.keys.begin(); it != data.keys.end(); ++it) { + Clearcut_System_Key key = *it; + if (key.short_name == short_name) { + return key.api_key; + } + } + return ""; + } + static size_t write_callback(void *contents, size_t size, size_t nmemb, void *userp) { + ((std::string *)userp)->append((char *)contents, size * nmemb); + return size * nmemb; + } + int upload(Call_Data_t call_info) { + + std::string api_key = get_api_key(call_info.short_name); + if (api_key.size() == 0) { + //BOOST_LOG_TRIVIAL(error) << "[" << call_info.short_name << "]\tTG: " << talkgroup_display << "\t " << std::put_time(std::localtime(&start_time), "%c %Z") << "\tClearCut Upload failed, API Key not found in config for shortName"; + return 0; + } + + std::ostringstream freq; + std::string freq_string; + freq << std::fixed << std::setprecision(0); + freq << call_info.freq; + + std::ostringstream call_length; + std::string call_length_string; + call_length << std::fixed << std::setprecision(0); + call_length << call_info.length; + + std::ostringstream source_list; + std::string source_list_string; + source_list << std::fixed << std::setprecision(2); + source_list << "["; + + if (call_info.transmission_source_list.size() != 0) { + for (int i = 0; i < call_info.transmission_source_list.size(); i++) { + source_list << "{\"src\": " << std::fixed << call_info.transmission_source_list[i].source << ", \"time\": " << call_info.transmission_source_list[i].time << ", \"pos\": " << std::fixed << std::setprecision(2) << call_info.transmission_source_list[i].position << ", \"emergency\": " << call_info.transmission_source_list[i].emergency << ", \"signal_system\": \"" << call_info.transmission_source_list[i].signal_system << "\", \"tag\": \"" << call_info.transmission_source_list[i].tag << "\"}"; + if (i < (call_info.transmission_source_list.size() - 1)) { + source_list << ", "; + } else { + source_list << "]"; + } + } + } else { + source_list << "]"; + } + + + std::ostringstream freq_list; + std::string freq_list_string; + freq_list << std::fixed << std::setprecision(2); + freq_list << "["; + + if (call_info.transmission_error_list.size() != 0) { + for (std::size_t i = 0; i < call_info.transmission_error_list.size(); i++) { + freq_list << "{\"freq\": " << std::fixed << std::setprecision(0) << call_info.freq << ", \"time\": " << call_info.transmission_error_list[i].time << ", \"pos\": " << std::fixed << std::setprecision(2) << call_info.transmission_error_list[i].position << ", \"len\": " << call_info.transmission_error_list[i].total_len << ", \"error_count\": \"" << std::setprecision(0) <(formattedTalkgroup); + CURL *curl; + CURLMcode res; + CURLM *multi_handle; + int still_running = 0; + std::string response_buffer; + freq_string = freq.str(); + + source_list_string = source_list.str(); + freq_list_string = freq_list.str(); + call_length_string = call_length.str(); + + struct curl_httppost *formpost = NULL; + struct curl_httppost *lastptr = NULL; + struct curl_slist *headerlist = NULL; + + /* Fill in the file upload field. This makes libcurl load data from + the given file name when curl_easy_perform() is called. */ + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "call", + CURLFORM_FILE, call_info.converted, + CURLFORM_CONTENTTYPE, "application/octet-stream", + CURLFORM_END); + + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "freq", + CURLFORM_COPYCONTENTS, freq_string.c_str(), + CURLFORM_END); + + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "start_time", + CURLFORM_COPYCONTENTS, boost::lexical_cast(call_info.start_time).c_str(), + CURLFORM_END); + + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "stop_time", + CURLFORM_COPYCONTENTS, boost::lexical_cast(call_info.stop_time).c_str(), + CURLFORM_END); + + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "call_length", + CURLFORM_COPYCONTENTS, call_length_string.c_str(), + CURLFORM_END); + + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "talkgroup_num", + CURLFORM_COPYCONTENTS, boost::lexical_cast(call_info.talkgroup).c_str(), + CURLFORM_END); + + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "emergency", + CURLFORM_COPYCONTENTS, boost::lexical_cast(call_info.emergency).c_str(), + CURLFORM_END); + + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "api_key", + CURLFORM_COPYCONTENTS, api_key.c_str(), + CURLFORM_END); + + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "source_list", + CURLFORM_COPYCONTENTS, source_list_string.c_str(), + CURLFORM_END); + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "freq_list", + CURLFORM_COPYCONTENTS, freq_list_string.c_str(), + CURLFORM_END); + + curl = curl_easy_init(); + multi_handle = curl_multi_init(); + + /* initialize custom header list (stating that Expect: 100-continue is not wanted */ + headerlist = curl_slist_append(headerlist, "Expect:"); + if (curl && multi_handle) { + std::string url = data.clearcut_server + "/api/v1/upload"; + + /* what URL that receives this POST */ + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + curl_easy_setopt(curl, CURLOPT_USERAGENT, "TrunkRecorder1.0"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist); + curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_buffer); + + curl_multi_add_handle(multi_handle, curl); + + curl_multi_perform(multi_handle, &still_running); + + while (still_running) { + struct timeval timeout; + int rc; /* select() return code */ + CURLMcode mc; /* curl_multi_fdset() return code */ + + fd_set fdread; + fd_set fdwrite; + fd_set fdexcep; + int maxfd = -1; + + long curl_timeo = -1; + + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcep); + + /* set a suitable timeout to play around with */ + timeout.tv_sec = 1; + timeout.tv_usec = 0; + + curl_multi_timeout(multi_handle, &curl_timeo); + if (curl_timeo >= 0) { + timeout.tv_sec = curl_timeo / 1000; + if (timeout.tv_sec > 1) + timeout.tv_sec = 1; + else + timeout.tv_usec = (curl_timeo % 1000) * 1000; + } + + /* get file descriptors from the transfers */ + mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); + + if (mc != CURLM_OK) { + fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc); + break; + } + + /* On success the value of maxfd is guaranteed to be >= -1. We call + select(maxfd + 1, ...); specially in case of (maxfd == -1) there are + no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- + to sleep 100ms, which is the minimum suggested value in the + curl_multi_fdset() doc. */ + + if (maxfd == -1) { + /* Portable sleep for platforms other than Windows. */ + struct timeval wait = {0, 100 * 1000}; /* 100ms */ + rc = select(0, NULL, NULL, NULL, &wait); + } else { + /* Note that on some platforms 'timeout' may be modified by select(). + If you need access to the original value save a copy beforehand. */ + rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); + } + + switch (rc) { + case -1: + /* select error */ + break; + case 0: + default: + /* timeout or readable/writable sockets */ + curl_multi_perform(multi_handle, &still_running); + break; + } + } + + long response_code; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + + res = curl_multi_cleanup(multi_handle); + + /* always cleanup */ + curl_easy_cleanup(curl); + + /* then cleanup the formpost chain */ + curl_formfree(formpost); + + /* free slist */ + curl_slist_free_all(headerlist); + + if (res == CURLM_OK && response_code == 200) { + struct stat file_info; + stat(call_info.converted, &file_info); + + BOOST_LOG_TRIVIAL(info) << "[" << call_info.short_name << "]\t\033[0;34m" << call_info.call_num << "C\033[0m\tTG: " << call_info.talkgroup_display << "\tFreq: " << format_freq(call_info.freq) << "\tClearCut Upload Success - file size: " << file_info.st_size; + ; + return 0; + } + } + BOOST_LOG_TRIVIAL(error) << "[" << call_info.short_name << "]\t\033[0;34m" << call_info.call_num << "C\033[0m\tTG: " << call_info.talkgroup_display << "\tFreq: " << format_freq(call_info.freq) << "\tClearCut Upload Error: " << response_buffer; + return 1; + } + + int call_end(Call_Data_t call_info) { + return upload(call_info); + } + + int parse_config(boost::property_tree::ptree &cfg) { + boost::optional upload_server_exists = cfg.get_optional("clearcutServer"); + if (!upload_server_exists) { + return 1; + } + + this->data.clearcut_server = cfg.get("clearcutServer", ""); + BOOST_LOG_TRIVIAL(info) << "ClearCut Server: " << this->data.clearcut_server; + + // from: http://www.zedwood.com/article/cpp-boost-url-regex + boost::regex ex("(http|https)://([^/ :]+):?([^/ ]*)(/?[^ #?]*)\\x3f?([^ #]*)#?([^ ]*)"); + boost::cmatch what; + + if (!regex_match(this->data.clearcut_server.c_str(), what, ex)) { + BOOST_LOG_TRIVIAL(error) << "Unable to parse Server URL\n"; + return 1; + } + // Gets the API key for each system, if defined + BOOST_FOREACH (boost::property_tree::ptree::value_type &node, cfg.get_child("systems")) { + boost::optional clearcut_exists = node.second.get_child_optional("clearcutApiKey"); + if (clearcut_exists) { + Clearcut_System_Key key; + key.api_key = node.second.get("clearcutApiKey", ""); + key.short_name = node.second.get("shortName", ""); + BOOST_LOG_TRIVIAL(info) << "Uploading calls to ClearCut for: " << key.short_name; + this->data.keys.push_back(key); + } + } + + if (this->data.keys.size() == 0) { + BOOST_LOG_TRIVIAL(error) << "ClearCut Server set, but no Systems are configured\n"; + return 1; + } + + return 0; + } + + /* + int init(Config *config, std::vector sources, std::vector systems) { return 0; } + int start() { return 0; } + int stop() { return 0; } + int poll_one() { return 0; } + int signal(long unitId, const char *signaling_type, gr::blocks::SignalType sig_type, Call *call, System *system, Recorder *recorder) { return 0; } + int audio_stream(Recorder *recorder, float *samples, int sampleCount) { return 0; } + int call_start(Call *call) { return 0; } + int calls_active(std::vector calls) { return 0; } + int setup_recorder(Recorder *recorder) { return 0; } + int setup_system(System *system) { return 0; } + int setup_systems(std::vector systems) { return 0; } + int setup_sources(std::vector sources) { return 0; } + int setup_config(std::vector sources, std::vector systems) { return 0; } + int system_rates(std::vector systems, float timeDiff) { return 0; } +*/ + // Factory method + static boost::shared_ptr create() { + return boost::shared_ptr( + new Clearcut_Uploader()); + } +}; + +BOOST_DLL_ALIAS( + Clearcut_Uploader::create, // <-- this function is exported with... + create_plugin // <-- ...this alias name +) diff --git a/trunk-recorder/main.cc b/trunk-recorder/main.cc index 7584cb81..2ebbf205 100644 --- a/trunk-recorder/main.cc +++ b/trunk-recorder/main.cc @@ -570,8 +570,9 @@ bool load_config(string config_file) { } BOOST_LOG_TRIVIAL(info) << "\n\n-------------------------------------\nPLUGINS\n-------------------------------------\n"; - add_internal_plugin("openmhz_uploader", "libopenmhz_uploader.so", pt); add_internal_plugin("broadcastify_uploader", "libbroadcastify_uploader.so", pt); + add_internal_plugin("openmhz_uploader", "libopenmhz_uploader.so", pt); + add_internal_plugin("clearcut_uploader", "libclearcut_uploader.so", pt); add_internal_plugin("unit_script", "libunit_script.so", pt); add_internal_plugin("stat_socket", "libstat_socket.so", pt); initialize_plugins(pt, &config, sources, systems); -- cgit v1.2.3