UrlClient.cpp

Go to the documentation of this file.

 
// UrlClient.cpp
 
// Implements the cUrlClient class for high-level URL interaction
 
#include "Globals.h"
 
#include "HTTP/UrlClient.h"
#include "HTTP/UrlParser.h"
#include "HTTP/HTTPMessageParser.h"
#include "mbedTLS++/X509Cert.h"
#include "mbedTLS++/CryptoKey.h"
 
 
 
 
 
// fwd:
class cSchemeHandler;
using cSchemeHandlerPtr = std::shared_ptr<cSchemeHandler>;
 
 
 
 
 
namespace
{
    class cBlockingHTTPCallbacks :
        public cUrlClient::cCallbacks
    {
    public:
 
        explicit cBlockingHTTPCallbacks(std::shared_ptr<cEvent> a_Event, AString & a_ResponseBody) :
            m_Event(std::move(a_Event)), m_ResponseBody(a_ResponseBody),
            m_IsError(false)
        {
        }
 
        void OnBodyFinished() override
        {
            m_Event->Set();
        }
 
        void OnError(const AString & a_ErrorMsg) override
        {
            LOGERROR("%s %d: HTTP Error: %s", __FILE__, __LINE__, a_ErrorMsg.c_str());
            m_IsError = true;
            m_Event->Set();
        }
 
        void OnBodyData(const void * a_Data, size_t a_Size) override
        {
            m_ResponseBody.append(static_cast<const char *>(a_Data), a_Size);
        }
 
        std::shared_ptr<cEvent> m_Event;
 
        AString & m_ResponseBody;
 
        bool m_IsError;
    };
}
 
 
 
 
 
class cUrlClientRequest:
    public cNetwork::cConnectCallbacks,
    public cTCPLink::cCallbacks
{
    friend class cHttpSchemeHandler;
 
public:
    static std::pair<bool, AString> Request(
        const AString & a_Method,
        const AString & a_URL,
        cUrlClient::cCallbacksPtr && a_Callbacks,
        AStringMap && a_Headers,
        const AString & a_Body,
        const AStringMap & a_Options
    )
    {
        // Create a new instance of cUrlClientRequest, wrapped in a SharedPtr, so that it has a controlled lifetime.
        // Cannot use std::make_shared, because the constructor is not public
        std::shared_ptr<cUrlClientRequest> ptr (new cUrlClientRequest(
            a_Method, a_URL, std::move(a_Callbacks), std::move(a_Headers), a_Body, a_Options
        ));
        return ptr->DoRequest(ptr);
    }
 
 
    void CallErrorCallback(const AString & a_ErrorMessage)
    {
        // Call the error callback:
        m_Callbacks->OnError(a_ErrorMessage);
 
        // Terminate the request's TCP link:
        if (auto link = m_Link.lock())
        {
            link->Close();
        }
    }
 
 
    cUrlClient::cCallbacks & GetCallbacks() { return *m_Callbacks; }
 
    void RedirectTo(const AString & a_RedirectUrl);
 
    bool ShouldAllowRedirects() const;
 
    cX509CertPtr GetOwnCert() const
    {
        auto itr = m_Options.find("OwnCert");
        if (itr == m_Options.end())
        {
            return nullptr;
        }
        cX509CertPtr cert = std::make_shared<cX509Cert>();
        if (!cert->Parse(itr->second.data(), itr->second.size()))
        {
            LOGD("OwnCert failed to parse");
            return nullptr;
        }
        return cert;
    }
 
    cCryptoKeyPtr GetOwnPrivKey() const
    {
        auto itr = m_Options.find("OwnPrivKey");
        if (itr == m_Options.end())
        {
            return nullptr;
        }
        cCryptoKeyPtr key = std::make_shared<cCryptoKey>();
        auto passItr = m_Options.find("OwnPrivKeyPassword");
        auto pass = (passItr == m_Options.end()) ? AString() : passItr->second;
        if (!key->ParsePrivate(itr->second.data(), itr->second.size(), pass))
        {
            return nullptr;
        }
        return key;
    }
 
    cX509CertPtr GetTrustedRootCAs() const
    {
        auto itr = m_Options.find("TrustedRootCAs");
        if (itr == m_Options.end())
        {
            return nullptr;
        }
        auto Cert = std::make_shared<cX509Cert>();
        auto Res = Cert->Parse(itr->second.data(), itr->second.size());
        if (Res != 0)
        {
            throw std::runtime_error(fmt::format("Failed to parse the TrustedRootCAs certificate: {}", Res));
        }
        return Cert;
    }
 
protected:
 
    AString m_Method;
 
    AString m_Url;
 
    AString m_UrlScheme, m_UrlUsername, m_UrlPassword, m_UrlHost, m_UrlPath, m_UrlQuery, m_UrlFragment;
    UInt16 m_UrlPort;
 
    cUrlClient::cCallbacksPtr m_Callbacks;
 
    AStringMap m_Headers;
 
    AString m_Body;
 
    AStringMap m_Options;
 
    std::weak_ptr<cUrlClientRequest> m_Self;
 
    std::shared_ptr<cSchemeHandler> m_SchemeHandler;
 
    std::weak_ptr<cTCPLink> m_Link;
 
    int m_NumRemainingRedirects;
 
 
    cUrlClientRequest(
        const AString & a_Method,
        const AString & a_Url,
        cUrlClient::cCallbacksPtr && a_Callbacks,
        AStringMap && a_Headers,
        const AString & a_Body,
        const AStringMap & a_Options
    ):
        m_Method(a_Method),
        m_Url(a_Url),
        m_Callbacks(std::move(a_Callbacks)),
        m_Headers(std::move(a_Headers)),
        m_Body(a_Body),
        m_Options(a_Options)
    {
        m_NumRemainingRedirects = GetStringMapInteger(m_Options, "MaxRedirects", 30);
    }
 
 
    std::pair<bool, AString> DoRequest(const std::shared_ptr<cUrlClientRequest> & a_Self);
 
 
    // cNetwork::cConnectCallbacks override: TCP link connected:
    virtual void OnConnected(cTCPLink & a_Link) override;
 
    // cNetwork::cConnectCallbacks override: An error has occurred:
    virtual void OnError(int a_ErrorCode, const AString & a_ErrorMsg) override
    {
        m_Callbacks->OnError(fmt::format(FMT_STRING("Network error {} ({})"), a_ErrorCode, a_ErrorMsg));
    }
 
 
    // cTCPLink::cCallbacks override: TCP link created
    virtual void OnLinkCreated(cTCPLinkPtr a_Link) override
    {
        m_Link = a_Link;
    }
 
 
    // cTCPLink::cCallbacks override: TLS handshake completed on the link:
    virtual void OnTlsHandshakeCompleted(void) override;
 
    virtual void OnReceivedData(const char * a_Data, size_t a_Length) override;
 
 
    virtual void OnRemoteClosed(void) override;
};
 
 
 
 
 
class cSchemeHandler abstract
{
public:
    cSchemeHandler(cUrlClientRequest & a_ParentRequest):
        m_ParentRequest(a_ParentRequest)
    {
    }
 
    // Force a virtual destructor in all descendants
    virtual ~cSchemeHandler() {}
 
    static cSchemeHandlerPtr Create(const AString & a_Scheme, cUrlClientRequest & a_ParentRequest);
 
    virtual void OnConnected(cTCPLink & a_Link) = 0;
 
    virtual void OnReceivedData(const char * a_Data, size_t a_Length) = 0;
 
    virtual void OnTlsHandshakeCompleted(void) = 0;
 
    virtual void OnRemoteClosed(void) = 0;
 
protected:
    cUrlClientRequest & m_ParentRequest;
};
 
 
 
 
 
class cHttpSchemeHandler:
    public cSchemeHandler,
    protected cHTTPMessageParser::cCallbacks
{
    using Super = cSchemeHandler;
 
public:
 
    cHttpSchemeHandler(cUrlClientRequest & a_ParentRequest, bool a_IsTls):
        Super(a_ParentRequest),
        m_Parser(*this),
        m_IsTls(a_IsTls),
        m_IsRedirect(false)
    {
    }
 
 
    virtual void OnConnected(cTCPLink & a_Link) override
    {
        m_Link = &a_Link;
        if (m_IsTls)
        {
            m_Link->StartTLSClient(m_ParentRequest.GetOwnCert(), m_ParentRequest.GetOwnPrivKey(), m_ParentRequest.GetTrustedRootCAs());
        }
        else
        {
            SendRequest();
        }
    }
 
 
    void SendRequest()
    {
        // Compose the request line:
        auto requestLine = m_ParentRequest.m_UrlPath;
        if (requestLine.empty())
        {
            requestLine = "/";
        }
        if (!m_ParentRequest.m_UrlQuery.empty())
        {
            requestLine.push_back('?');
            requestLine.append(m_ParentRequest.m_UrlQuery);
        }
        m_Link->Send(fmt::format(FMT_STRING("{} {} HTTP/1.1\r\n"), m_ParentRequest.m_Method, requestLine));
 
        // Send the headers:
        m_Link->Send(fmt::format(FMT_STRING("Host: {}\r\n"), m_ParentRequest.m_UrlHost));
        m_Link->Send(fmt::format(FMT_STRING("Content-Length: {}\r\n"), m_ParentRequest.m_Body.size()));
        for (const auto & hdr: m_ParentRequest.m_Headers)
        {
            m_Link->Send(fmt::format(FMT_STRING("{}: {}\r\n"), hdr.first, hdr.second));
        }  // for itr - m_Headers[]
        m_Link->Send("\r\n", 2);
 
        // Send the body:
        m_Link->Send(m_ParentRequest.m_Body);
 
        // Notify the callbacks that the request has been sent:
        m_ParentRequest.GetCallbacks().OnRequestSent();
    }
 
 
    virtual void OnReceivedData(const char * a_Data, size_t a_Length) override
    {
        auto res = m_Parser.Parse(a_Data, a_Length);
        if (res == AString::npos)
        {
            m_ParentRequest.CallErrorCallback("Failed to parse HTTP response");
            return;
        }
    }
 
 
    virtual void OnTlsHandshakeCompleted(void) override
    {
        SendRequest();
    }
 
 
    virtual void OnRemoteClosed(void) override
    {
        m_Link = nullptr;
    }
 
 
    // cHTTPResponseParser::cCallbacks overrides:
    virtual void OnError(const AString & a_ErrorDescription) override
    {
        m_Link = nullptr;
        m_ParentRequest.CallErrorCallback(a_ErrorDescription);
    }
 
 
    virtual void OnFirstLine(const AString & a_FirstLine) override
    {
        // Find the first space, parse the result code between it and the second space:
        auto idxFirstSpace = a_FirstLine.find(' ');
        if (idxFirstSpace == AString::npos)
        {
            m_ParentRequest.CallErrorCallback(fmt::format(FMT_STRING("Failed to parse HTTP status line \"{}\", no space delimiter."), a_FirstLine));
            return;
        }
        auto idxSecondSpace = a_FirstLine.find(' ', idxFirstSpace + 1);
        if (idxSecondSpace == AString::npos)
        {
            m_ParentRequest.CallErrorCallback(fmt::format(FMT_STRING("Failed to parse HTTP status line \"{}\", missing second space delimiter."), a_FirstLine));
            return;
        }
        int resultCode;
        auto resultCodeStr = a_FirstLine.substr(idxFirstSpace + 1, idxSecondSpace - idxFirstSpace - 1);
        if (!StringToInteger(resultCodeStr, resultCode))
        {
            m_ParentRequest.CallErrorCallback(fmt::format(FMT_STRING("Failed to parse HTTP result code from response \"{}\""), resultCodeStr));
            return;
        }
 
        // Check for redirects, follow if allowed by the options:
        switch (resultCode)
        {
            case cUrlClient::HTTP_STATUS_MULTIPLE_CHOICES:
            case cUrlClient::HTTP_STATUS_MOVED_PERMANENTLY:
            case cUrlClient::HTTP_STATUS_FOUND:
            case cUrlClient::HTTP_STATUS_SEE_OTHER:
            case cUrlClient::HTTP_STATUS_TEMPORARY_REDIRECT:
            {
                m_IsRedirect = true;
                return;
            }
        }
        m_ParentRequest.GetCallbacks().OnStatusLine(a_FirstLine.substr(0, idxFirstSpace), resultCode, a_FirstLine.substr(idxSecondSpace + 1));
    }
 
 
    virtual void OnHeaderLine(const AString & a_Key, const AString & a_Value) override
    {
        if (m_IsRedirect)
        {
            if (a_Key == "Location")
            {
                m_RedirectLocation = a_Value;
            }
        }
        else
        {
            m_ParentRequest.GetCallbacks().OnHeader(a_Key, a_Value);
        }
    }
 
 
    virtual void OnHeadersFinished(void) override
    {
        if (!m_IsRedirect)
        {
            m_ParentRequest.GetCallbacks().OnHeadersFinished();
        }
    }
 
 
    virtual void OnBodyData(const void * a_Data, size_t a_Size) override
    {
        if (!m_IsRedirect)
        {
            m_ParentRequest.GetCallbacks().OnBodyData(a_Data, a_Size);
        }
    }
 
 
    virtual void OnBodyFinished(void) override
    {
        if (m_IsRedirect)
        {
            if (m_RedirectLocation.empty())
            {
                m_ParentRequest.CallErrorCallback("Invalid redirect, there's no location to redirect to");
            }
            else
            {
                m_ParentRequest.RedirectTo(m_RedirectLocation);
            }
        }
        else
        {
            m_ParentRequest.GetCallbacks().OnBodyFinished();
            // Finished recieving data, shutdown the link
            m_Link->Shutdown();
        }
    }
 
protected:
 
    cTCPLink * m_Link;
 
    cHTTPMessageParser m_Parser;
 
    bool m_IsTls;
 
    bool m_IsRedirect;
 
    AString m_RedirectLocation;
};
 
 
 
 
 
// cSchemeHandler:
 
cSchemeHandlerPtr cSchemeHandler::Create(const AString & a_Scheme, cUrlClientRequest & a_ParentRequest)
{
    auto lowerScheme = StrToLower(a_Scheme);
    if (lowerScheme == "http")
    {
        return std::make_shared<cHttpSchemeHandler>(a_ParentRequest, false);
    }
    else if (lowerScheme == "https")
    {
        return std::make_shared<cHttpSchemeHandler>(a_ParentRequest, true);
    }
 
    return nullptr;
}
 
 
 
 
 
// cUrlClientRequest:
 
void cUrlClientRequest::RedirectTo(const AString & a_RedirectUrl)
{
    // Check that redirection is allowed:
    m_Callbacks->OnRedirecting(a_RedirectUrl);
    if (!ShouldAllowRedirects())
    {
        CallErrorCallback(fmt::format(FMT_STRING("Redirect to \"{}\" not allowed"), a_RedirectUrl));
        return;
    }
 
    // Keep ourself alive while the link drops ownership
    auto Self = m_Self.lock();
 
    // Do the actual redirect:
    if (auto Link = m_Link.lock())
    {
        Link->Close();
    }
 
    m_Url = a_RedirectUrl;
    m_NumRemainingRedirects = m_NumRemainingRedirects - 1;
    auto res = DoRequest(Self);
    if (!res.first)
    {
        m_Callbacks->OnError(fmt::format(FMT_STRING("Redirection failed: {}"), res.second));
    }
}
 
 
 
 
 
bool cUrlClientRequest::ShouldAllowRedirects() const
{
    return (m_NumRemainingRedirects > 0);
}
 
 
 
 
 
void cUrlClientRequest::OnConnected(cTCPLink & a_Link)
{
    m_Callbacks->OnConnected(a_Link);
    m_SchemeHandler->OnConnected(a_Link);
}
 
 
 
 
 
void cUrlClientRequest::OnTlsHandshakeCompleted(void)
{
    // Notify the scheme handler and the callbacks:
    m_SchemeHandler->OnTlsHandshakeCompleted();
    m_Callbacks->OnTlsHandshakeCompleted();
}
 
 
 
 
 
void cUrlClientRequest::OnReceivedData(const char * a_Data, size_t a_Length)
{
    auto handler = m_SchemeHandler;
    if (handler != nullptr)
    {
        handler->OnReceivedData(a_Data, a_Length);
    }
}
 
 
 
 
 
void cUrlClientRequest::OnRemoteClosed()
{
    // Notify the callback:
    auto handler = m_SchemeHandler;
    if (handler != nullptr)
    {
        handler->OnRemoteClosed();
    }
}
 
 
 
 
 
std::pair<bool, AString> cUrlClientRequest::DoRequest(const std::shared_ptr<cUrlClientRequest> & a_Self)
{
    // We need a shared pointer to self, care must be taken not to pass any other ptr:
    ASSERT(a_Self.get() == this);
 
    m_Self = a_Self;
 
    // Parse the URL:
    auto res = cUrlParser::Parse(m_Url, m_UrlScheme, m_UrlUsername, m_UrlPassword, m_UrlHost, m_UrlPort, m_UrlPath, m_UrlQuery, m_UrlFragment);
    if (!res.first)
    {
        return res;
    }
 
    // Get a handler that will work with the specified scheme:
    m_SchemeHandler = cSchemeHandler::Create(m_UrlScheme, *this);
    if (m_SchemeHandler == nullptr)
    {
        return std::make_pair(false, fmt::format(FMT_STRING("Unknown URL scheme: {}"), m_UrlScheme));
    }
 
    // Connect and transfer ownership to the link
    if (!cNetwork::Connect(m_UrlHost, m_UrlPort, a_Self, a_Self))
    {
        return std::make_pair(false, "Network connection failed");
    }
    return std::make_pair(true, AString());
}
 
 
 
 
 
// cUrlClient:
 
std::pair<bool, AString> cUrlClient::Request(
    const AString & a_Method,
    const AString & a_URL,
    cCallbacksPtr && a_Callbacks,
    AStringMap && a_Headers,
    const AString & a_Body,
    const AStringMap & a_Options
)
{
    return cUrlClientRequest::Request(
        a_Method, a_URL, std::move(a_Callbacks), std::move(a_Headers), a_Body, a_Options
    );
}
 
 
 
 
 
std::pair<bool, AString> cUrlClient::Get(
    const AString & a_URL,
    cCallbacksPtr && a_Callbacks,
    AStringMap && a_Headers,
    const AString & a_Body,
    const AStringMap & a_Options
)
{
    return cUrlClientRequest::Request(
        "GET", a_URL, std::move(a_Callbacks), std::move(a_Headers), a_Body, a_Options
    );
}
 
 
 
 
 
std::pair<bool, AString> cUrlClient::Post(
    const AString & a_URL,
    cCallbacksPtr && a_Callbacks,
    AStringMap && a_Headers,
    const AString & a_Body,
    const AStringMap & a_Options
)
{
    return cUrlClientRequest::Request(
        "POST", a_URL, std::move(a_Callbacks), std::move(a_Headers), a_Body, a_Options
    );
}
 
 
 
 
 
std::pair<bool, AString> cUrlClient::Put(
    const AString & a_URL,
    cCallbacksPtr && a_Callbacks,
    AStringMap && a_Headers,
    const AString & a_Body,
    const AStringMap & a_Options
)
{
    return cUrlClientRequest::Request(
        "PUT", a_URL, std::move(a_Callbacks), std::move(a_Headers), a_Body, a_Options
    );
}
 
 
 
 
 
std::pair<bool, AString> cUrlClient::BlockingRequest(
    const AString & a_Method,
    const AString & a_URL,
    AStringMap && a_Headers,
    const AString & a_Body,
    const AStringMap & a_Options
)
{
    auto EvtFinished = std::make_shared<cEvent>();
    AString Response;
    auto Callbacks = std::make_shared<cBlockingHTTPCallbacks>(EvtFinished, Response);
    auto [Success, ErrorMessage] = cUrlClient::Request(a_Method, a_URL, Callbacks, std::move(a_Headers), a_Body, a_Options);
    if (Success)
    {
        if (!EvtFinished->Wait(10000))
        {
            return std::make_pair(false, "Timeout");
        }
        if (Callbacks->m_IsError)
        {
            return std::make_pair(false, AString());
        }
    }
    else
    {
        LOGWARNING("%s: HTTP error: %s", __FUNCTION__, ErrorMessage.c_str());
        return std::make_pair(false, AString());
    }
    return std::make_pair(true, Response);
}
 
 
 
 
 
std::pair<bool, AString> cUrlClient::BlockingGet(
    const AString & a_URL,
    AStringMap a_Headers,
    const AString & a_Body,
    const AStringMap & a_Options
)
{
    return BlockingRequest("GET", a_URL, std::move(a_Headers), a_Body, a_Options);
}
 
 
 
 
 
std::pair<bool, AString> cUrlClient::BlockingPost(
    const AString & a_URL,
    AStringMap && a_Headers,
    const AString & a_Body,
    const AStringMap & a_Options
)
{
    return BlockingRequest("POST", a_URL, std::move(a_Headers), a_Body, a_Options);
}
 
 
 
 
 
std::pair<bool, AString> cUrlClient::BlockingPut(
    const AString & a_URL,
    AStringMap && a_Headers,
    const AString & a_Body,
    const AStringMap & a_Options
)
{
    return BlockingRequest("PUT", a_URL, std::move(a_Headers), a_Body, a_Options);
}