Mod hashing + better download (#374)

include/Client.h

@@ -66,7 +66,6 @@ public:
     std::string GetCarData(int Ident);
     std::string GetCarPositionRaw(int Ident);
     void SetUDPAddr(const ip::udp::endpoint& Addr) { mUDPAddress = Addr; }
-    void SetDownSock(ip::tcp::socket&& CSock) { mDownSocket = std::move(CSock); }
     void SetTCPSock(ip::tcp::socket&& CSock) { mSocket = std::move(CSock); }
     void Disconnect(std::string_view Reason);
     bool IsDisconnected() const { return !mSocket.is_open(); }
@@ -75,8 +74,6 @@ public:
     [[nodiscard]] const std::unordered_map<std::string, std::string>& GetIdentifiers() const { return mIdentifiers; }
     [[nodiscard]] const ip::udp::endpoint& GetUDPAddr() const { return mUDPAddress; }
     [[nodiscard]] ip::udp::endpoint& GetUDPAddr() { return mUDPAddress; }
-    [[nodiscard]] ip::tcp::socket& GetDownSock() { return mDownSocket; }
-    [[nodiscard]] const ip::tcp::socket& GetDownSock() const { return mDownSocket; }
     [[nodiscard]] ip::tcp::socket& GetTCPSock() { return mSocket; }
     [[nodiscard]] const ip::tcp::socket& GetTCPSock() const { return mSocket; }
     [[nodiscard]] std::string GetRoles() const { return mRole; }
@@ -122,7 +119,6 @@ private:
     SparseArray<std::string> mVehiclePosition;
     std::string mName = "Unknown Client";
     ip::tcp::socket mSocket;
-    ip::tcp::socket mDownSocket;
     ip::udp::endpoint mUDPAddress {};
     int mUnicycleID = -1;
     std::string mRole;

include/TNetwork.h

@@ -58,7 +58,6 @@ private:
     std::mutex mOpenIDMutex;
 
     std::vector<uint8_t> UDPRcvFromClient(ip::udp::endpoint& ClientEndpoint);
-    void HandleDownload(TConnection&& TCPSock);
     void OnConnect(const std::weak_ptr<TClient>& c);
     void TCPClient(const std::weak_ptr<TClient>& c);
     void Looper(const std::weak_ptr<TClient>& c);
@@ -67,7 +66,7 @@ private:
     void Parse(TClient& c, const std::vector<uint8_t>& Packet);
     void SendFile(TClient& c, const std::string& Name);
     static bool TCPSendRaw(TClient& C, ip::tcp::socket& socket, const uint8_t* Data, size_t Size);
-    static void SplitLoad(TClient& c, size_t Sent, size_t Size, bool D, const std::string& Name);
+    static void SendFileToClient(TClient& c, size_t Size, const std::string& Name);
     static const uint8_t* SendSplit(TClient& c, ip::tcp::socket& Socket, const uint8_t* DataPtr, size_t Size);
 };
 

include/TResourceManager.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include "Common.h"
+#include <nlohmann/json.hpp>
 
 class TResourceManager {
 public:
@@ -30,10 +31,17 @@ public:
     [[nodiscard]] std::string FileSizes() const { return mFileSizes; }
     [[nodiscard]] int ModsLoaded() const { return mModsLoaded; }
 
+    [[nodiscard]] std::string NewFileList() const;
+
+    void RefreshFiles();
+
 private:
     size_t mMaxModSize = 0;
     std::string mFileSizes;
     std::string mFileList;
     std::string mTrimmedList;
     int mModsLoaded = 0;
+
+    std::mutex mModsMutex;
+    nlohmann::json mMods;
 };

src/Client.cpp

@@ -144,7 +144,6 @@ void TClient::EnqueuePacket(const std::vector<uint8_t>& Packet) {
 TClient::TClient(TServer& Server, ip::tcp::socket&& Socket)
     : mServer(Server)
     , mSocket(std::move(Socket))
-    , mDownSocket(ip::tcp::socket(Server.IoCtx()))
     , mLastPingTime(std::chrono::high_resolution_clock::now()) {
 }
 

src/TNetwork.cpp

@@ -240,7 +240,8 @@ void TNetwork::Identify(TConnection&& RawConnection) {
         if (Code == 'C') {
             Client = Authentication(std::move(RawConnection));
         } else if (Code == 'D') {
-            HandleDownload(std::move(RawConnection));
+            beammp_errorf("Old download packet detected - the client is wildly out of date, this will be ignored");
+            return;
         } else if (Code == 'P') {
             boost::system::error_code ec;
             write(RawConnection.Socket, buffer("P"), ec);
@@ -249,7 +250,7 @@ void TNetwork::Identify(TConnection&& RawConnection) {
             beammp_errorf("Invalid code got in Identify: '{}'", Code);
         }
     } catch (const std::exception& e) {
-        beammp_errorf("Error during handling of code {} - client left in invalid state, closing socket", Code);
+        beammp_errorf("Error during handling of code {} - client left in invalid state, closing socket: {}", Code, e.what());
         boost::system::error_code ec;
         RawConnection.Socket.shutdown(socket_base::shutdown_both, ec);
         if (ec) {
@@ -262,27 +263,7 @@ void TNetwork::Identify(TConnection&& RawConnection) {
     }
 }
 
-void TNetwork::HandleDownload(TConnection&& Conn) {
+
-    char D;
-    boost::system::error_code ec;
-    read(Conn.Socket, buffer(&D, 1), ec);
-    if (ec) {
-        Conn.Socket.shutdown(socket_base::shutdown_both, ec);
-        // ignore ec
-        return;
-    }
-    auto ID = uint8_t(D);
-    mServer.ForEachClient([&](const std::weak_ptr<TClient>& ClientPtr) -> bool {
-        ReadLock Lock(mServer.GetClientMutex());
-        if (!ClientPtr.expired()) {
-            auto c = ClientPtr.lock();
-            if (c->GetID() == ID) {
-                c->SetDownSock(std::move(Conn.Socket));
-            }
-        }
-        return true;
-    });
-}
 
 std::string HashPassword(const std::string& str) {
     std::stringstream ret;
@@ -758,11 +739,11 @@ void TNetwork::Parse(TClient& c, const std::vector<uint8_t>& Packet) {
     case 'S':
         if (SubCode == 'R') {
             beammp_debug("Sending Mod Info");
-            std::string ToSend = mResourceManager.FileList() + mResourceManager.FileSizes();
+            std::string ToSend = mResourceManager.NewFileList();
-            if (ToSend.empty())
+            beammp_debugf("Mod Info: {}", ToSend);
-                ToSend = "-";
             if (!TCPSend(c, StringToVector(ToSend))) {
-                // TODO: error
+                ClientKick(c, "TCP Send 'SY' failed");
+                return;
             }
         }
         return;
@@ -772,8 +753,6 @@ void TNetwork::Parse(TClient& c, const std::vector<uint8_t>& Packet) {
 }
 
 void TNetwork::SendFile(TClient& c, const std::string& UnsafeName) {
-    beammp_info(c.GetName() + " requesting : " + UnsafeName.substr(UnsafeName.find_last_of('/')));
-
     if (!fs::path(UnsafeName).has_filename()) {
         if (!TCPSend(c, StringToVector("CO"))) {
             // TODO: handle
@@ -796,87 +775,9 @@ void TNetwork::SendFile(TClient& c, const std::string& UnsafeName) {
         // TODO: handle
     }
 
-    /// Wait for connections
-    int T = 0;
-    while (!c.GetDownSock().is_open() && T < 50) {
-        std::this_thread::sleep_for(std::chrono::milliseconds(100));
-        T++;
-    }
-
-    if (!c.GetDownSock().is_open()) {
-        beammp_error("Client doesn't have a download socket!");
-        if (!c.IsDisconnected())
-            c.Disconnect("Missing download socket");
-        return;
-    }
-
     size_t Size = size_t(std::filesystem::file_size(FileName));
-    size_t MSize = Size / 2;
 
-    std::thread SplitThreads[2] {
+    SendFileToClient(c, Size, FileName);
-        std::thread([&] {
-            RegisterThread("SplitLoad_0");
-            SplitLoad(c, 0, MSize, false, FileName);
-        }),
-        std::thread([&] {
-            RegisterThread("SplitLoad_1");
-            SplitLoad(c, MSize, Size, true, FileName);
-        })
-    };
-
-    for (auto& SplitThread : SplitThreads) {
-        if (SplitThread.joinable()) {
-            SplitThread.join();
-        }
-    }
-}
-
-static std::pair<size_t /* count */, size_t /* last chunk */> SplitIntoChunks(size_t FullSize, size_t ChunkSize) {
-    if (FullSize < ChunkSize) {
-        return { 0, FullSize };
-    }
-    size_t Count = FullSize / (FullSize / ChunkSize);
-    size_t LastChunkSize = FullSize - (Count * ChunkSize);
-    return { Count, LastChunkSize };
-}
-
-TEST_CASE("SplitIntoChunks") {
-    size_t FullSize;
-    size_t ChunkSize;
-    SUBCASE("Normal case") {
-        FullSize = 1234567;
-        ChunkSize = 1234;
-    }
-    SUBCASE("Zero original size") {
-        FullSize = 0;
-        ChunkSize = 100;
-    }
-    SUBCASE("Equal full size and chunk size") {
-        FullSize = 125;
-        ChunkSize = 125;
-    }
-    SUBCASE("Even split") {
-        FullSize = 10000;
-        ChunkSize = 100;
-    }
-    SUBCASE("Odd split") {
-        FullSize = 13;
-        ChunkSize = 2;
-    }
-    SUBCASE("Large sizes") {
-        FullSize = 10 * GB;
-        ChunkSize = 125 * MB;
-    }
-    auto [Count, LastSize] = SplitIntoChunks(FullSize, ChunkSize);
-    CHECK((Count * ChunkSize) + LastSize == FullSize);
-}
-
-const uint8_t* /* end ptr */ TNetwork::SendSplit(TClient& c, ip::tcp::socket& Socket, const uint8_t* DataPtr, size_t Size) {
-    if (TCPSendRaw(c, Socket, DataPtr, Size)) {
-        return DataPtr + Size;
-    } else {
-        return nullptr;
-    }
 }
 
 #if defined(BEAMMP_LINUX)
@@ -886,8 +787,8 @@ const uint8_t* /* end ptr */ TNetwork::SendSplit(TClient& c, ip::tcp::socket& So
 #include <unistd.h>
 #include <signal.h>
 #endif
-void TNetwork::SplitLoad(TClient& c, size_t Offset, size_t End, bool D, const std::string& Name) {
+void TNetwork::SendFileToClient(TClient& c, size_t Size, const std::string& Name) {
-    TScopedTimer timer(fmt::format("Download of {}-{} for '{}'", Offset, End, Name));
+    TScopedTimer timer(fmt::format("Download of '{}' for client {}", Name, c.GetID()));
 #if defined(BEAMMP_LINUX)
     signal(SIGPIPE, SIG_IGN);
     // on linux, we can use sendfile(2)!
@@ -897,11 +798,11 @@ void TNetwork::SplitLoad(TClient& c, size_t Offset, size_t End, bool D, const st
         return;
     }
     // native handle, needed in order to make native syscalls with it
-    int socket = D ? c.GetDownSock().native_handle() : c.GetTCPSock().native_handle();
+    int socket = c.GetTCPSock().native_handle();
 
     ssize_t ret = 0;
-    auto ToSendTotal = End - Offset;
+    auto ToSendTotal = Size;
-    auto Start = Offset;
+    auto Start = 0;
     while (ret < ssize_t(ToSendTotal)) {
         auto SysOffset = off_t(Start + size_t(ret));
         ret = sendfile(socket, fd, &SysOffset, ToSendTotal - size_t(ret));
@@ -915,35 +816,32 @@ void TNetwork::SplitLoad(TClient& c, size_t Offset, size_t End, bool D, const st
     std::ifstream f(Name.c_str(), std::ios::binary);
     uint32_t Split = 125 * MB;
     std::vector<uint8_t> Data;
-    if (End > Split)
+    if (Size > Split)
         Data.resize(Split);
     else
-        Data.resize(End);
+        Data.resize(Size);
-    ip::tcp::socket* TCPSock { nullptr };
+    ip::tcp::socket* TCPSock = &c.GetTCPSock();
-    if (D)
+    std::streamsize Sent = 0;
-        TCPSock = &c.GetDownSock();
+    while (!c.IsDisconnected() && Sent < Size) {
-    else
+        size_t Diff = Size - Sent;
-        TCPSock = &c.GetTCPSock();
-    while (!c.IsDisconnected() && Offset < End) {
-        size_t Diff = End - Offset;
         if (Diff > Split) {
-            f.seekg(Offset, std::ios_base::beg);
+            f.seekg(Sent, std::ios_base::beg);
             f.read(reinterpret_cast<char*>(Data.data()), Split);
             if (!TCPSendRaw(c, *TCPSock, Data.data(), Split)) {
                 if (!c.IsDisconnected())
                     c.Disconnect("TCPSendRaw failed in mod download (1)");
                 break;
             }
-            Offset += Split;
+            Sent += Split;
         } else {
-            f.seekg(Offset, std::ios_base::beg);
+            f.seekg(Sent, std::ios_base::beg);
             f.read(reinterpret_cast<char*>(Data.data()), Diff);
             if (!TCPSendRaw(c, *TCPSock, Data.data(), int32_t(Diff))) {
                 if (!c.IsDisconnected())
                     c.Disconnect("TCPSendRaw failed in mod download (2)");
                 break;
             }
-            Offset += Diff;
+            Sent += Diff;
         }
     }
 #endif

src/TResourceManager.cpp

@@ -17,9 +17,14 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 #include "TResourceManager.h"
+#include "Common.h"
 
 #include <algorithm>
 #include <filesystem>
+#include <fmt/core.h>
+#include <ios>
+#include <nlohmann/json.hpp>
+#include <openssl/evp.h>
 
 namespace fs = std::filesystem;
 
@@ -52,3 +57,83 @@ TResourceManager::TResourceManager() {
 
     Application::SetSubsystemStatus("ResourceManager", Application::Status::Good);
 }
+
+std::string TResourceManager::NewFileList() const {
+    return mMods.dump();
+}
+void TResourceManager::RefreshFiles() {
+    mMods.clear();
+    std::unique_lock Lock(mModsMutex);
+
+    std::string Path = Application::Settings.getAsString(Settings::Key::General_ResourceFolder) + "/Client";
+    for (const auto& entry : fs::directory_iterator(Path)) {
+        std::string File(entry.path().string());
+
+        if (entry.path().extension() != ".zip" || std::filesystem::is_directory(entry.path())) {
+            beammp_warnf("'{}' is not a ZIP file and will be ignored", File);
+            continue;
+        }
+
+        try {
+            EVP_MD_CTX* mdctx;
+            const EVP_MD* md;
+            uint8_t sha256_value[EVP_MAX_MD_SIZE];
+            md = EVP_sha256();
+            if (md == nullptr) {
+                throw std::runtime_error("EVP_sha256() failed");
+            }
+
+            mdctx = EVP_MD_CTX_new();
+            if (mdctx == nullptr) {
+                throw std::runtime_error("EVP_MD_CTX_new() failed");
+            }
+            if (!EVP_DigestInit_ex2(mdctx, md, NULL)) {
+                EVP_MD_CTX_free(mdctx);
+                throw std::runtime_error("EVP_DigestInit_ex2() failed");
+            }
+
+            std::ifstream stream(File, std::ios::binary);
+
+            const size_t FileSize = std::filesystem::file_size(File);
+            size_t Read = 0;
+            std::vector<char> Data;
+            while (Read < FileSize) {
+                Data.resize(size_t(std::min<size_t>(FileSize - Read, 4096)));
+                size_t RealDataSize = Data.size();
+                stream.read(Data.data(), std::streamsize(Data.size()));
+                if (stream.eof() || stream.fail()) {
+                    RealDataSize = size_t(stream.gcount());
+                }
+                Data.resize(RealDataSize);
+                if (RealDataSize == 0) {
+                    break;
+                }
+                if (RealDataSize > 0 && !EVP_DigestUpdate(mdctx, Data.data(), Data.size())) {
+                    EVP_MD_CTX_free(mdctx);
+                    throw std::runtime_error("EVP_DigestUpdate() failed");
+                }
+                Read += RealDataSize;
+            }
+            unsigned int sha256_len = 0;
+            if (!EVP_DigestFinal_ex(mdctx, sha256_value, &sha256_len)) {
+                EVP_MD_CTX_free(mdctx);
+                throw std::runtime_error("EVP_DigestFinal_ex() failed");
+            }
+            EVP_MD_CTX_free(mdctx);
+
+            std::string result;
+            for (size_t i = 0; i < sha256_len; i++) {
+                result += fmt::format("{:02x}", sha256_value[i]);
+            }
+            beammp_debugf("sha256('{}'): {}", File, result);
+            mMods.push_back(nlohmann::json {
+                { "file_name", std::filesystem::path(File).filename() },
+                { "file_size", std::filesystem::file_size(File) },
+                { "hash_algorithm", "sha256" },
+                { "hash", result },
+            });
+        } catch (const std::exception& e) {
+            beammp_errorf("Sha256 hashing of '{}' failed: {}", File, e.what());
+        }
+    }
+}

src/main.cpp

@@ -36,19 +36,19 @@
 #include <thread>
 
 static const std::string sCommandlineArguments = R"(
-USAGE: 
+USAGE:
     BeamMP-Server [arguments]
-    
+
 ARGUMENTS:
-    --help              
+    --help
                         Displays this help and exits.
     --port=1234
                         Sets the server's listening TCP and
                         UDP port. Overrides ENV and ServerConfig.
     --config=/path/to/ServerConfig.toml
-                        Absolute or relative path to the 
+                        Absolute or relative path to the
                         Server Config file, including the
-                        filename. For paths and filenames with 
+                        filename. For paths and filenames with
                         spaces, put quotes around the path.
     --working-directory=/path/to/folder
                         Sets the working directory of the Server.
@@ -59,7 +59,7 @@ ARGUMENTS:
 
 EXAMPLES:
     BeamMP-Server --config=../MyWestCoastServerConfig.toml
-        Runs the BeamMP-Server and uses the server config file 
+        Runs the BeamMP-Server and uses the server config file
         which is one directory above it and is named
         'MyWestCoastServerConfig.toml'.
 )";
@@ -190,6 +190,7 @@ int BeamMPServerMain(MainArguments Arguments) {
 
     beammp_trace("Running in debug mode on a debug build");
     TResourceManager ResourceManager;
+    ResourceManager.RefreshFiles();
     TPPSMonitor PPSMonitor(Server);
     THeartbeatThread Heartbeat(ResourceManager, Server);
     TNetwork Network(Server, PPSMonitor, ResourceManager);