diff options
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | TODO | 2 | ||||
| -rw-r--r-- | plugins/statistics/Makefile | 124 | ||||
| -rw-r--r-- | plugins/statistics/statistics.cpp | 97 | ||||
| -rw-r--r-- | plugins/statistics/statistics.h | 21 | ||||
| -rw-r--r-- | response.cpp | 63 | ||||
| -rw-r--r-- | server.cpp | 15 | ||||
| -rw-r--r-- | statistics.cpp | 69 | ||||
| -rw-r--r-- | statistics.h | 5 | ||||
| -rw-r--r-- | webserver.conf | 4 |
10 files changed, 353 insertions, 49 deletions
@@ -1,7 +1,7 @@ DISTROS=debian10 VERSION=$(shell dpkg-parsechangelog --show-field Version) PROJECTNAME=webserver -PLUGINS=static-files webbox cgi weblog # fcgi +PLUGINS=static-files webbox cgi weblog statistics # fcgi CXX=clang++-10 @@ -1,3 +1,4 @@ +Upload: read: body limit exceeded weblog: blättern weblog: link consistency check (cron?) weblog: style: zitate @@ -5,7 +6,6 @@ Integrate into Debian Ubuntu version Speed up config.GetPath read: The socket was closed due to a timeout -print statistics webbox: Info if not selected: all webbox: Copy function crypt pws diff --git a/plugins/statistics/Makefile b/plugins/statistics/Makefile new file mode 100644 index 0000000..48c2e8c --- /dev/null +++ b/plugins/statistics/Makefile @@ -0,0 +1,124 @@ +DISTROS=debian10 +VERSION=$(shell dpkg-parsechangelog --show-field Version) +PROJECTNAME=statistics + +CXX=clang++-10 + +ifeq ($(shell which $(CXX)),) +CXX=clang++ +endif + +ifeq ($(shell which $(CXX)),) +CXX=g++-9 +endif + +ifeq ($(CXXFLAGS),) +#CXXFLAGS=-O2 -DNDEBUG +CXXFLAGS=-O0 -g -D_DEBUG +endif +# -fprofile-instr-generate -fcoverage-mapping +# gcc:--coverage + +CXXFLAGS+= -Wall -I. + +CXXFLAGS+= -pthread -fvisibility=hidden -fPIC +ifeq ($(CXX),clang++-10) +CXXFLAGS+=-std=c++20 #-stdlib=libc++ +else +CXXFLAGS+=-std=c++17 +endif + +CXXTESTFLAGS=-Igoogletest/include -Igooglemock/include/ -Igoogletest -Igooglemock + +LIBS=\ +-lboost_context \ +-lboost_coroutine \ +-lboost_program_options \ +-lboost_system \ +-lboost_thread \ +-lboost_filesystem \ +-lboost_regex \ +-lpthread \ +-lssl -lcrypto \ +-ldl + +ifeq ($(CXX),clang++-10) +LIBS+= \ +-fuse-ld=lld-10 \ +-lstdc++ +#-lc++ \ +#-lc++abi +#-lc++fs +#-lstdc++fs +else +LIBS+= \ +-lstdc++ \ +-lstdc++fs +endif + +PROGSRC=\ + statistics.cpp + +TESTSRC=\ + test-webserver.cpp \ + googlemock/src/gmock-all.cpp \ + googletest/src/gtest-all.cpp \ + $(PROGSRC) + +SRC=$(PROGSRC) + +all: $(PROJECTNAME).so + +# testsuite ---------------------------------------------- +test-$(PROJECTNAME): $(TESTSRC:.cpp=.o) + $(CXX) $(CXXFLAGS) $^ $(LIBS) -o $@ + +$(PROJECTNAME).so: $(SRC:.cpp=.o) + $(CXX) -shared $(CXXFLAGS) $^ $(LIBS) -o $@ + +dep: $(TESTSRC:.cpp=.d) + +%.d: %.cpp + $(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -MM -MP -MF $@ -c $< + +%.o: %.cpp %.d + $(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -c $< -o $@ + +googletest/src/%.o: googletest/src/%.cc + $(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -c $< -o $@ + +# dependencies + +ADD_DEP=Makefile + +install: + mkdir -p $(DESTDIR)/usr/lib/webserver/plugins + cp $(PROJECTNAME).so $(DESTDIR)/usr/lib/webserver/plugins + +# misc --------------------------------------------------- +deb: + # build binary deb package + dpkg-buildpackage -us -uc -rfakeroot + +deb-src: + dpkg-source -b . + +$(DISTROS): deb-src + sudo pbuilder build --basetgz /var/cache/pbuilder/$@.tgz --buildresult result/$@ ../webserver_$(VERSION).dsc ; \ + +debs: $(DISTROS) + +clean: + -rm -f test-$(PROJECTNAME) $(PROJECTNAME) + -find . -name '*.o' -o -name '*.so' -o -name '*.d' -o -name '*.gcno' -o -name '*.gcda' | xargs rm -f + +zip: clean + -rm -f ../$(PROJECTNAME).zip + zip -r ../$(PROJECTNAME).zip * + ls -l ../$(PROJECTNAME).zip + + + +.PHONY: clean all zip install deb deb-src debs all $(DISTROS) + +-include $(wildcard $(SRC:.cpp=.d)) diff --git a/plugins/statistics/statistics.cpp b/plugins/statistics/statistics.cpp new file mode 100644 index 0000000..03f4c94 --- /dev/null +++ b/plugins/statistics/statistics.cpp @@ -0,0 +1,97 @@ +#include "statistics.h" + +#include <boost/algorithm/string/predicate.hpp> +#include <boost/coroutine2/coroutine.hpp> +#include <boost/process.hpp> + +#include <algorithm> +#include <filesystem> +#include <fstream> +#include <iostream> +#include <string> +#include <unordered_map> + +using namespace std::string_literals; +namespace bp = boost::process; +namespace fs = std::filesystem; + +namespace { + + // Used to return errors by generating response page and HTTP status code + std::string HttpStatus(std::string status, std::string message, std::function<plugin_interface_setter_type>& SetResponseHeader) + { + SetResponseHeader("status", status); + SetResponseHeader("content_type", "text/html"); + return status + " " + message; + } + +} // anonymous namespace + +std::string statistics_plugin::name() +{ + return "statistics"; +} + +statistics_plugin::statistics_plugin() +{ + //std::cout << "Plugin constructor" << std::endl; +} + +statistics_plugin::~statistics_plugin() +{ + //std::cout << "Plugin destructor" << std::endl; +} + +std::string statistics_plugin::generate_page( + std::function<std::string(const std::string& key)>& GetServerParam, + std::function<std::string(const std::string& key)>& GetRequestParam, // request including body (POST...) + std::function<void(const std::string& key, const std::string& value)>& SetResponseHeader // to be added to result string +) +{ + try { + // Request path must not contain "..". + std::string rel_target{GetRequestParam("rel_target")}; + size_t query_pos{rel_target.find("?")}; + if (query_pos != rel_target.npos) + rel_target = rel_target.substr(0, query_pos); + + std::string target{GetRequestParam("target")}; + if (rel_target.find("..") != std::string::npos) { + return HttpStatus("400", "Illegal request: "s + target, SetResponseHeader); + } + + // Build the path to the requested file + std::string doc_root{GetRequestParam("doc_root")}; + fs::path path {fs::path{doc_root} / rel_target}; + if (target.size() && target.back() != '/' && fs::is_directory(path)) { + std::string location{GetRequestParam("location") + "/"s}; + SetResponseHeader("location", location); + return HttpStatus("301", "Correcting directory path", SetResponseHeader); + } + + try { + SetResponseHeader("content_type", "text/html"); + + std::string header {"<!DOCTYPE html><html><head>" + "<meta charset=\"utf-8\"/>" + "<title>Webserver Statistics</title>" + "</head><body>"}; + std::string footer{"<br/><br/><br/></body></html>"}; + + std::string result{header}; + + result += "<h1>Webserver Statistics</h1>"; + result += "<pre>"s + GetServerParam("statistics") + "</pre>"s; + + result += footer; + + return result; + } catch (const std::exception& ex) { + return HttpStatus("500", "Statistics error: "s + ex.what(), SetResponseHeader); + } + + } catch (const std::exception& ex) { + return HttpStatus("500", "Unknown Error: "s + ex.what(), SetResponseHeader); + } +} + diff --git a/plugins/statistics/statistics.h b/plugins/statistics/statistics.h new file mode 100644 index 0000000..5db309b --- /dev/null +++ b/plugins/statistics/statistics.h @@ -0,0 +1,21 @@ +#pragma once + +#include "../../plugin_interface.h" + +class statistics_plugin: public webserver_plugin_interface +{ +public: + statistics_plugin(); + ~statistics_plugin(); + + std::string name(); + std::string generate_page( + std::function<std::string(const std::string& key)>& GetServerParam, + std::function<std::string(const std::string& key)>& GetRequestParam, // request including body (POST...) + std::function<void(const std::string& key, const std::string& value)>& SetResponseHeader // to be added to result string + ); + +}; + +extern "C" BOOST_SYMBOL_EXPORT statistics_plugin webserver_plugin; +statistics_plugin webserver_plugin; diff --git a/response.cpp b/response.cpp index c5ba426..696b859 100644 --- a/response.cpp +++ b/response.cpp @@ -79,10 +79,11 @@ bool is_ipv6_address(const std::string& addr) std::unordered_map<std::string, std::function<std::string(Server&)>> GetServerParamFunctions{ // following are the supported fields: - {"version", [](Server& server) { return Server::VersionString; }}, {"address", [](Server& server) { return server.GetSocket().address; }}, {"ipv6", [](Server& server) { return is_ipv6_address(server.GetSocket().address) ? "yes" : "no"; }}, {"port", [](Server& server) { return server.GetSocket().port; }}, + {"statistics", [](Server& server) { return server.GetStatistics().getValues(); }}, + {"version", [](Server& server) { return Server::VersionString; }}, }; std::string GetServerParam(const std::string& key, Server& server) @@ -233,42 +234,34 @@ mime_type(beast::string_view path) return "application/text"; } -// Used to return errors by generating response page and HTTP status code response_type HttpStatus(std::string status, std::string message, response_type& res) { - res.result(unsigned(stoul(status))); - res.set(http::field::content_type, "text/html"); - if (res.result_int() == 401) - res.set(http::field::www_authenticate, "Basic realm=\"Webbox Login\""); - res.body() = "<html><body><h1>"s + Server::VersionString + " Error</h1><p>"s + status + " "s + message + "</p></body></html>"s; - res.prepare_payload(); + if (status != "200") { // already handled at res init + res.result(unsigned(stoul(status))); + res.set(http::field::content_type, "text/html"); + if (res.result_int() == 401) + res.set(http::field::www_authenticate, "Basic realm=\"Webbox Login\""); + res.body() = "<html><body><h1>"s + Server::VersionString + " Error</h1><p>"s + status + " "s + message + "</p></body></html>"s; + res.prepare_payload(); + } return res; } -// Do statistics at end of response generation, handle all exit paths via RAII -class StatisticsGuard +// Used to return errors by generating response page and HTTP status code +response_type HttpStatusAndStats(std::string status, std::string message, RequestContext& req_ctx, response_type& res) { - request_type& mReq; - response_type& mRes; - Server& mServer; -public: - StatisticsGuard(request_type& req, response_type& res, Server& server) - : mReq(req) - , mRes(res) - , mServer(server) - { - } + HttpStatus(status, message, res); - ~StatisticsGuard() - { - mServer.GetStatistics().count(mReq.body().size(), - mRes.body().size(), - mRes.result_int() == 200, - is_ipv6_address(mServer.GetSocket().address), - mServer.GetSocket().protocol == SocketProtocol::HTTPS); - } -}; + req_ctx.GetServer().GetStatistics().count( + req_ctx.GetReq().body().size(), + res.body().size(), + res.result_int() != 200, + is_ipv6_address(req_ctx.GetServer().GetSocket().address), + req_ctx.GetServer().GetSocket().protocol == SocketProtocol::HTTPS); + + return std::move(res); +} } // anonymous namespace @@ -279,8 +272,6 @@ response_type generate_response(request_type& req, Server& server) res.set(http::field::content_type, mime_type(extend_index_html(std::string(req.target())))); res.keep_alive(req.keep_alive()); - StatisticsGuard statsGuard{req, res, server}; - try { RequestContext req_ctx{req, server}; // can throw std::out_of_range @@ -288,21 +279,21 @@ response_type generate_response(request_type& req, Server& server) if (auth.size() != 0) { std::string authorization{req[http::field::authorization]}; if (authorization.substr(0, 6) != "Basic "s) - return HttpStatus("401", "Bad Authorization Type", res); + return HttpStatusAndStats("401", "Bad Authorization Type", req_ctx, res); authorization = authorization.substr(6); authorization = decode64(authorization); size_t pos {authorization.find(':')}; if (pos == authorization.npos) - return HttpStatus("401", "Bad Authorization Encoding", res); + return HttpStatusAndStats("401", "Bad Authorization Encoding", req_ctx, res); std::string login{authorization.substr(0, pos)}; std::string password{authorization.substr(pos + 1)}; auto it {auth.find(login)}; if (it == auth.end() || it->second != password) - return HttpStatus("401", "Bad Authorization", res); + return HttpStatusAndStats("401", "Bad Authorization", req_ctx, res); } plugin_type plugin{req_ctx.GetPlugin()}; @@ -318,8 +309,8 @@ response_type generate_response(request_type& req, Server& server) res.body() = res_data; res.prepare_payload(); } - - return res; + + return HttpStatusAndStats("200", "OK", req_ctx, res); } catch(const std::out_of_range& ex) { return HttpStatus("400", "Bad request: Host "s + std::string{req["host"]} + ":"s + std::string{req.target()} + " unknown"s, res); } catch(const std::exception& ex) { @@ -20,6 +20,7 @@ #include <boost/config.hpp> #include <exception> +#include <functional> #include <iostream> #include <thread> #include <vector> @@ -39,6 +40,10 @@ using tcp = boost::asio::ip::tcp; // from <boost/asio/ip/tcp.hpp> const std::string Server::VersionString{ "Reichwein.IT Webserver "s + std::string{VERSION} }; +namespace { + const int32_t stats_timer_seconds { 24 * 60 * 60 }; // save stats once a day +} // anonymous namespace + Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics) : m_config(config) , m_ioc(ioc) @@ -66,6 +71,16 @@ int run_server(Config& config, plugins_container_type& plugins) ioc.stop(); }); + // Save stats once a day + boost::asio::steady_timer stats_save_timer(ioc, boost::asio::chrono::seconds(stats_timer_seconds)); + std::function<void(const boost::system::error_code&)> stats_callback = + [&](const boost::system::error_code& error){ + stats.save(); + stats_save_timer.expires_at(stats_save_timer.expires_at() + boost::asio::chrono::seconds(stats_timer_seconds)); + stats_save_timer.async_wait(stats_callback); + }; + stats_save_timer.async_wait(stats_callback); + std::vector<std::shared_ptr<Server>> servers; const auto& sockets {config.Sockets()}; diff --git a/statistics.cpp b/statistics.cpp index 19d0258..3fb99a3 100644 --- a/statistics.cpp +++ b/statistics.cpp @@ -5,13 +5,15 @@ #include <iostream> namespace fs = std::filesystem; +using namespace std::string_literals; namespace { const fs::path statsfilepath{ "/var/lib/webserver/stats.db" }; } // anonymous namespace -Statistics::Statistics() +void Statistics::load() { + std::lock_guard<std::mutex> lock(mMutex); std::cout << "Loading statistics..." << std::endl; std::ifstream file{statsfilepath, std::ios::in | std::ios::binary}; if (file.is_open()) { @@ -21,22 +23,38 @@ Statistics::Statistics() } else { std::cerr << "Warning: Couldn't read statistics" << std::endl; } + + mChanged = false; } -Statistics::~Statistics() +void Statistics::save() { - std::cout << "Saving statistics..." << std::endl; - std::lock_guard<std::mutex> lock(mMutex); - std::ofstream file{statsfilepath, std::ios::out | std::ios::binary | std::ios::trunc}; - if (file.is_open()) { - Serialization::OArchive archive{file}; - - archive << mBins; - } else { - std::cerr << "Warning: Couldn't write statistics" << std::endl; + if (mChanged) { + std::lock_guard<std::mutex> lock(mMutex); + std::cout << "Saving statistics..." << std::endl; + std::ofstream file{statsfilepath, std::ios::out | std::ios::binary | std::ios::trunc}; + if (file.is_open()) { + Serialization::OArchive archive{file}; + + archive << mBins; + } else { + std::cerr << "Warning: Couldn't write statistics" << std::endl; + } + + mChanged = false; } } +Statistics::Statistics() +{ + load(); +} + +Statistics::~Statistics() +{ + save(); +} + bool Statistics::Bin::expired() const { auto now {time(nullptr)}; @@ -57,6 +75,8 @@ void Statistics::count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, { std::lock_guard<std::mutex> lock(mMutex); + mChanged = true; + if (mBins.empty() || mBins.back().expired()) { mBins.emplace_back(Bin{static_cast<uint64_t>((time(nullptr) / binsize) * binsize), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); } @@ -85,3 +105,30 @@ void Statistics::count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, limit(); } +std::string Statistics::getValues() +{ + std::lock_guard<std::mutex> lock(mMutex); + + std::string result; + + for (const auto& bin: mBins) { + result += std::to_string(bin.start_time) + ","s + + + std::to_string(bin.requests) + ","s + + std::to_string(bin.errors) + ","s + + std::to_string(bin.bytes_in) + ","s + + std::to_string(bin.bytes_out) + ","s + + + std::to_string(bin.requests_ipv6) + ","s + + std::to_string(bin.errors_ipv6) + ","s + + std::to_string(bin.bytes_in_ipv6) + ","s + + std::to_string(bin.bytes_out_ipv6) + ","s + + + std::to_string(bin.requests_https) + ","s + + std::to_string(bin.errors_https) + ","s + + std::to_string(bin.bytes_in_https) + ","s + + std::to_string(bin.bytes_out_https) + "\n"s; + } + + return result; +} diff --git a/statistics.h b/statistics.h index c4fce93..7e4da7e 100644 --- a/statistics.h +++ b/statistics.h @@ -60,9 +60,11 @@ public: }; private: + bool mChanged{}; std::deque<Bin> mBins; std::mutex mMutex; + void load(); void limit(); public: @@ -70,6 +72,9 @@ public: ~Statistics(); void count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, bool https); + void save(); + + std::string getValues(); }; // Serialization and Deserialization as free functions diff --git a/webserver.conf b/webserver.conf index 52075a2..5adc9ba 100644 --- a/webserver.conf +++ b/webserver.conf @@ -38,6 +38,10 @@ <WEBLOG_DESCRIPTION>Roland Reichweins Blog</WEBLOG_DESCRIPTION> <WEBLOG_KEYWORDS>Roland Reichwein, Blog</WEBLOG_KEYWORDS> </path> + <path requested="/stats"> + <plugin>statistics</plugin> + <target></target> + </path> <path requested="/cgi-bin"> <plugin>cgi</plugin> <target>/home/ernie/code/webserver/cgi-bin</target> |