/*---------------------------------------------------------------------------*/
/* http_client.cpp                                                           */
/* copyright (c) innovaphone 2015                                            */
/*                                                                           */
/*---------------------------------------------------------------------------*/

#include <platform.h>

#include <stdexcept>
#include <string>

#include "common/ilib/uri.h"
#include "common/ilib/base64.h"
#include "common/ilib/hash.h"
#include "common/ilib/str.h"
#include "sdk/common/lib/httplib.h"
#include "sdk/common/os/iomux.h"
#include "sdk/common/interface/dns.h"
#include "sdk/common/interface/random.h"
#include "sdk/common/interface/time.h"
#include "sdk/common/interface/socket.h"
#include "sdk/common/interface/network.h"
#include "sdk/common/interface/http_client.h"

#include "ntlm.h"
#include "http_client.h"

#define HTTPCLIENT_DEFAULT_TIMEOUT  60000

struct http_request_type_info
{
	HTTPLib::http_request_method_t method;
	std::string methodStr;
};

http_request_type_info RequstTypeToRequestInfo(http_request_type_t & requestType)
{
	http_request_type_info result;
	
	switch (requestType)
	{
	case HTTP_GET: 
		result.method = HTTPLib::HTTP_GET;
		result.methodStr = HTTP_GET_KEY;
		break;

	case HTTP_POST: 
		result.method = HTTPLib::HTTP_POST;
		result.methodStr = HTTP_POST_KEY;
		break;

	case HTTP_PUT: 
		result.method = HTTPLib::HTTP_PUT;
		result.methodStr = HTTP_PUT_KEY;
		break;

	case HTTP_DELETE:
		result.method = HTTPLib::HTTP_DELETE;
		result.methodStr = HTTP_DELETE_KEY;
		break;

	case HTTP_HEAD:
		result.method = HTTPLib::HTTP_HEAD;
		result.methodStr = HTTP_HEAD_KEY;
		break;

	case HTTP_OPTIONS:
		result.method = HTTPLib::HTTP_OPTIONS;
		result.methodStr = HTTP_OPTIONS_KEY;
		break;
    
	case HTTP_TRACE:
		result.method = HTTPLib::HTTP_TRACE;
		result.methodStr = HTTP_TRACE_KEY;
		break;
    
	case HTTP_PROPFIND:
		result.method = HTTPLib::HTTP_PROPFIND;
		result.methodStr = HTTP_PROPFIND_KEY;
		break;
    
	case HTTP_MKCOL:
		result.method = HTTPLib::HTTP_MKCOL;
		result.methodStr = HTTP_MKCOL_KEY;
		break;
    
	case HTTP_COPY:
		result.method = HTTPLib::HTTP_COPY;
		result.methodStr = HTTP_COPY_KEY;
		break;
    
	case HTTP_MOVE:
		result.method = HTTPLib::HTTP_MOVE;
		result.methodStr = HTTP_MOVE_KEY;
		break;
    
	case HTTP_LOCK:
		result.method = HTTPLib::HTTP_LOCK;
		result.methodStr = HTTP_LOCK_KEY;
		break;
    
	case HTTP_UNLOCK:
		result.method = HTTPLib::HTTP_UNLOCK;
		result.methodStr = HTTP_UNLOCK_KEY;
		break;
    
	case HTTP_PROPPATCH:
		result.method = HTTPLib::HTTP_PROPPATCH;
		result.methodStr = HTTP_PROPPATCH_KEY;
		break;
        
	case HTTP_PATCH:
		result.method = HTTPLib::HTTP_PATCH;
		result.methodStr = HTTP_PATCH_KEY;
		break;
	}
	
	return result;
}


inline void ReleaseCustomHeaderFields(CustomField *& fields)
{
    while (fields != nullptr) {
        CustomField * next = fields->next;
        delete fields;
        fields = next;
    }
}

/*---------------------------------------------------------------------------*/
/* class HTTPRedirectData                                                    */
/*---------------------------------------------------------------------------*/

    //redirect_type_t redirectType;
    //bool permanentRedirect = false;
    //http_request_type_t requestMethod = http_request_type_t::HTTP_GET;
    //std::string server;
    //std::string resource;
    //std::string contentType;
    //size_t contentLength = 0;
    //CustomField * customHeaderFields = nullptr;
    //byte * userBuffer = nullptr;
    //size_t * userBufferSize = 0;

HTTPRedirectData::HTTPRedirectData(bool permanent)
    : permanentRedirect(permanent)
{
}

HTTPRedirectData::~HTTPRedirectData()
{
    ReleaseCustomHeaderFields(this->customHeaderFields);
}

void HTTPRedirectData::Type(redirect_type_t rt)
{
    this->redirectType = rt;
}

void HTTPRedirectData::Location(const std::string & server, const std::string & resource)
{
    this->server = server;
    this->resource = resource;
}

void HTTPRedirectData::RequestData(http_request_type_t reqMethod, const char * contentType, size_t contLen)
{
    this->requestMethod = reqMethod;
    if (contentType != nullptr) this->contentType = contentType;
    this->contentLength = contLen;
}

void HTTPRedirectData::AuthData(const char * userName, const char * pwd, int authMethod)
{
    if (userName != nullptr) this->authUser = userName;
    if (pwd != nullptr) this->authPwd = pwd;
    this->authMethod = authMethod;
}

void HTTPRedirectData::CustomHeaderFields(CustomField * field)
{
    this->customHeaderFields = field;
}

void HTTPRedirectData::UserBuffer(byte * buf, size_t len, bool partitial)
{
    this->userBuffer = buf;
    this->userBufferSize = len;
    this->recvPartitial = partitial;
}


/*---------------------------------------------------------------------------*/
/* class IHTTPClient                                                         */
/*---------------------------------------------------------------------------*/

IHTTPClient * IHTTPClient::Create(class IIoMux * const iomux,
                                  class ISocketProvider * const tcpSocketProvider,
                                  class ISocketProvider * const tlsSocketProvider,
                                  class UHTTPClient * const user,
                                  class IInstanceLog * const log,
                                  class IDns * const dns,
                                  class ISocketContext * const socketContext)
{
    IRandom::Init(dword(ITime::TimeStampMilliseconds() / 1000));
    return new HTTPClient(iomux, tcpSocketProvider, tlsSocketProvider, user, log, dns, socketContext);
}


/*---------------------------------------------------------------------------*/
/* class HTTPClient                                                          */
/*---------------------------------------------------------------------------*/

HTTPClient::HTTPClient(class IIoMux * const iomux,
                       class ISocketProvider * const tcpSocketProvider,
                       class ISocketProvider * const tlsSocketProvider,
                       class UHTTPClient * const user,
                       class IInstanceLog * const log,
                       class IDns * const dns,
                       class ISocketContext * const socketContext)
    : iomux(iomux),
      log(log),
      user(user),
      timer(iomux, this)
{
    this->tcpSocketProvider = tcpSocketProvider;
    this->tlsSocketProvider = tlsSocketProvider;
    this->socket = nullptr;
    this->dns = dns;
    this->socketContext = socketContext;
    this->dnsSpecified = dns != nullptr;
    this->customFieldList = nullptr;

    this->connected = false;
    this->recvPartial = false;
    this->useTLS = false;
    this->consumingData = false;
    this->port = 80;
    this->hostName = nullptr;
    this->hostIPAddress[0] = '\0';
    this->proxyIPAddress[0] = '\0';

    this->authUserName = nullptr;
    this->authUserPassword = nullptr;
    this->authMethods = 0;
    this->requestType = HTTP_GET;
    this->resource = nullptr;
    this->contentType = nullptr;
    this->chunkedTransfer = false;
    this->chunkedTransferCalled = false;
    this->retryAuthenticated = false;
    this->fakingAuth = false;
    this->ntlmAuthenticated = false;
    this->dnsRetried = false;
    this->proxies = nullptr;
    this->proxy = nullptr;

    this->httpRedirectData = nullptr;

    this->lastResult = HTTP_RESULT_OK;
    this->shutdownReason = HTTP_SHUTDOWN_NORMAL;
    this->httpParser = new HTTPParser();
    this->httpHeader = new HTTPHeader();

    this->headerBuffer[0] = '\0';
    this->headerBufferDataStart = nullptr;
    this->headerBufferDataSize = 0;
    this->userBuffer = nullptr;
    this->userBufferSize = 0;
    this->savedUserBuffer = nullptr;
    this->savedUserBufferSize = 0;
    this->savedState = HTTP_CL_STATE_OFFLINE;
    this->savedLastResult = HTTP_RESULT_OK;

    this->state = HTTP_CL_STATE_OFFLINE;
    this->execCnt = 0;
    this->sendCnt = 0;
    
    this->answerContentLength = 0;
    this->responseCode = 0;
    this->responseCodeDescription = nullptr;
    this->contentLength = 0;
    this->bytesToRead = 0;
    this->bytesToSend = 0;
    this->userBufferAlreadyRead = 0;
    this->readUntilConnectionClose = false;
    memset(this->chunkSizeNum, 0, sizeof(this->chunkSizeNum));

    this->dataToSend = nullptr;
    this->dataToSendSize = 0;
    this->dataLast = false;
    this->passErrorToUserFlags = 0;
    this->authToRead = 0;
    this->sendContentForAuth = false;

    this->timeoutMs = HTTPCLIENT_DEFAULT_TIMEOUT;
    this->timeoutOnConnect = false;

    memset(&digest, 0, sizeof(digest));

    log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::HTTPClient", this, this->user);
}


HTTPClient::~HTTPClient()
{
    log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::~HTTPClient", this, this->user);
    if (this->hostIPAddress != this->hostName) free(this->hostName);
    free(this->authUserName);
    free(this->authUserPassword);
    free(this->dataToSend);
    free(this->resource);
    free(this->contentType);
    free(this->responseCodeDescription);
    SetDigest(nullptr, nullptr, nullptr, nullptr, nullptr, false);  // this frees the digest params

    delete this->httpRedirectData;
    ReleaseCustomHeaderFields(this->customFieldList);

    if (!this->dnsSpecified && this->dns) delete this->dns;
    delete this->socket;
    delete this->httpParser;
    delete this->httpHeader;
    if (this->proxies) delete this->proxies;
}


void HTTPClient::Connect(const char * address, const char * user, const char * pwd, int authMethods)
{
    log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::Connect address:%s user:%s pwd:%08x authMethods:%i", this, this->user, address, user, pwd, authMethods);
    ASSERT(state == HTTP_CL_STATE_OFFLINE, "Called HTTPClient::Connect() while client is not offline");
    ASSERT(address != nullptr, "HTTPClient::Connect() called with address == nullptr");
    ASSERT(*address != '\0', "HTTPClient::Connect() called with empty address string");
    
    // Because the HTTP Client can be reused, the shutdown-reason must be reset to not have a false shutdown reason value the next time.
    if (this->hostIPAddress != this->hostName) {
        free(this->hostName);
        this->hostName = nullptr;
    }

    this->hostIPAddress[0] = '\0';
    this->proxyIPAddress[0] = '\0';

    free(this->authUserName);
    this->authUserName = nullptr;
    free(this->authUserPassword);
    this->authUserPassword = nullptr;
    this->authMethods = authMethods;

    if (user && *user) this->authUserName = _strdup(user);
    if (pwd && *pwd) this->authUserPassword = _strdup(pwd);
    
    // If the URI is wrong, we will call an SetExec of the iomux object to inform the
    // user later of that error. WIth this we will keep the asynchron design.
    uri_dissector uriDisp;
    if (!uriDisp.dissect_uri((char *)address) || (!uriDisp.ip && (uriDisp.e_fqdn == uriDisp.fqdn))) { // Avoid ASSERT in Dns::GetHostByName() due to empty host
        this->shutdownReason = HTTP_ADDRESS_INVALID;
        this->Shutdown();
        return;
    }

    if ((!user || !*user) && uriDisp.usr) {
        free(this->authUserName);
        size_t usrLen = uriDisp.e_usr - uriDisp.usr;
        this->authUserName = (char *)malloc(usrLen + 1);
        memcpy(this->authUserName, uriDisp.usr, usrLen);
        this->authUserName[usrLen] = '\0';
        str::from_url(this->authUserName);
    }
    if ((!pwd || !*pwd) && uriDisp.pwd) {
        free(this->authUserPassword);
        size_t pwdLen = uriDisp.e_pwd - uriDisp.pwd;
        this->authUserPassword = (char *)malloc(pwdLen + 1);
        memcpy(this->authUserPassword, uriDisp.pwd, pwdLen);
        this->authUserPassword[pwdLen] = '\0';
        str::from_url(this->authUserPassword);
    }

    if (uriDisp.scheme && (uriDisp.e_scheme - uriDisp.scheme) == 5 &&
        (uriDisp.scheme[4] == 's' || uriDisp.scheme[4] == 'S')) {
        this->useTLS = true;
        this->port = 443;
    }

    if (uriDisp.port) this->port = atoi(uriDisp.port);

    // uri_despense has an HostName or an IP Address, never both of them...
    if (uriDisp.ip) {
        char *ipStart = uriDisp.ip;
        int len = uriDisp.e_ip - uriDisp.ip;
        if (*ipStart == '[') {
            ipStart++;
            len -= 2;
        }
        this->hostName = this->hostIPAddress;
        memcpy(this->hostIPAddress, ipStart, len);
        this->hostIPAddress[len] = 0;
    }
    else {
        int len = uriDisp.e_fqdn - uriDisp.fqdn;
        this->hostName = (char *)malloc(len + 1);
        memcpy(this->hostName, uriDisp.fqdn, len);
        this->hostName[len] = 0;
    }

    if (this->proxies) delete(this->proxies);
    this->proxies = INetwork::GetHTTPProxiesForURL(address);
    this->proxy = this->proxies->httpProxies.empty() ? nullptr : this->proxies->httpProxies.front();
    ConnectProxy();
}

void HTTPClient::ConnectProxy()
{
    while (this->proxy && (this->proxy->type != HTTP_PROXY_TYPE_HTTP) && (this->proxy->type != HTTP_PROXY_TYPE_HTTPS)) {
        this->proxy = this->proxy->goNext();
    }
    if (this->proxy) {
        char * p = 0;
        str::to_ip(this->proxy->host, &p, /*prefix*/0);
        if (p != this->proxy->host) {
            memcpy(this->proxyIPAddress, this->proxy->host, p - this->proxy->host);
            this->proxyIPAddress[p - this->proxy->host] = 0;
        }
    }
    if ((this->proxy && this->proxyIPAddress[0]) || (!this->proxy && this->hostIPAddress[0])) {
        this->dnsRetried = true;
        this->ConnectSocket();
    }
    else {
        this->dnsRetried = false;
        if (this->dns == nullptr) this->dns = IDns::Create(this->iomux);
        this->ResolveDns();
    }
}


void HTTPClient::ResolveDns()
{
    this->shutdownReason = HTTP_SHUTDOWN_NORMAL;
    this->dns->GetHostByName(this->proxy ? this->proxy->host : this->hostName, this, this->log, DNS_DEFAULT_TIMEOUT, false, this->proxy ? this->proxy->port : this->port);
    this->state = HTTP_CL_STATE_RESOLVE_HOST_NAME;
    this->StartTimer();
}

void HTTPClient::ConnectSocket()
{
    char socketAddress[100];

    this->shutdownReason = HTTP_SHUTDOWN_NORMAL;
    if (this->proxy) {
        bool useIPv6 = strchr(this->proxyIPAddress, ':') != nullptr;
        this->socket = this->tcpSocketProvider->CreateSocket(this->iomux, this, this->log, useIPv6, this->socketContext);
        _snprintf(socketAddress, 100, useIPv6 ? "[%s]:%u" : "%s:%u", this->proxyIPAddress, (this->proxy->port != 0) ? this->proxy->port : 80);
        this->state = HTTP_CL_STATE_CONNECT_TUNNEL;
    }
    else {
        bool useIPv6 = strchr(this->hostIPAddress, ':') != nullptr;
        this->socket = (this->useTLS ? this->tlsSocketProvider : this->tcpSocketProvider)->CreateSocket(this->iomux, this, this->log, useIPv6, this->socketContext);
        _snprintf(socketAddress, 100, useIPv6 ? "[%s]:%u" : "%s:%u", this->hostIPAddress, this->port);
        this->state = HTTP_CL_STATE_RESOLVED_HOST_NAME;
    }
    this->socket->Connect(socketAddress, this->hostName);
    this->StartTimer();
}


void HTTPClient::AdjustBufferCounter(size_t diff)
{
    if (!this->readUntilConnectionClose) this->bytesToRead -= diff;
    this->userBufferAlreadyRead += diff;
}


bool HTTPClient::ConnectionIsKeelAlive()
{
    // Check for Keep-Alive flag - it depends on the value given to the connection field and the HTTP version used.
    // For HTTP 1.0, the default is to close the connection if keepalive is not given. For HTTP 1.1, the default is
    // to not close the connection, until the connection field had been set to close.
    if (this->httpParser->GetVersion() == HTTP_VERSION_1_0) {
        return this->httpParser->IsConnectionType(HTTPLib::HTTP_CONNECTION_KEEP_ALIVE);
    }
    else {
        return !this->httpParser->IsConnectionType(HTTPLib::HTTP_CONNECTION_CLOSE);
    }
}


void HTTPClient::DoRecv()
{
    switch (this->state)
    {
    case HTTP_CL_STATE_HEADER_SENT:
    case HTTP_CL_STATE_READ_HEADER:
        // do nothing, read should be already running
        break;

    case HTTP_CL_STATE_START_READ_CONTENT:
        StartReadRequestData();
        break;

    case HTTP_CL_STATE_READ_CONTENT:
    case HTTP_CL_STATE_READ_CHUNKED:
        if (this->headerBufferDataSize > 0) this->CopyDataAlreadyLoaded();
        else this->DoSocketRecv(this->userBuffer, this->bytesToRead < this->userBufferSize ? this->bytesToRead : this->userBufferSize, this->recvPartial); // this->readUntilConnectionClose || this->recvPartial);
        break;

    case HTTP_CL_STATE_READ_CHUNK_SIZE:
        StartReadChunk();
        break;

    default:
        break;
    }
}


void HTTPClient::DoSocketRecv(void * buf, size_t len, bool recvPartial)
{
    this->socket->Recv(buf, len, recvPartial);
    this->StartTimer();
}


void HTTPClient::Recv(byte * buffer, size_t size, bool recvPartial)
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::Recv size:%i, recvPartial:%s", this, this->user, size, recvPartial ? "TRUE" : "FALSE");
    this->recvPartial = recvPartial;
    if (this->consumingData) {
        ASSERT(this->savedUserBuffer == nullptr, "HTTPClient::Recv() called with already set userBuffer");
        this->savedUserBuffer = buffer;
        this->savedUserBufferSize = size;
    }
    else {
        ASSERT(this->userBuffer == nullptr, "HTTPClient::Recv() called with already set userBuffer");
        this->userBuffer = buffer;
        this->userBufferSize = size;
        this->userBufferAlreadyRead = 0;

        switch (this->state)
        {
        case HTTP_CL_STATE_HEADER_SENT:
        case HTTP_CL_STATE_READ_HEADER:
            // do nothing, read should be already running
            break;

        case HTTP_CL_STATE_START_READ_CONTENT:
            this->iomux->SetExec(this, nullptr);
            break;

        case HTTP_CL_STATE_READ_CONTENT:
        case HTTP_CL_STATE_READ_CHUNKED:
            if (this->headerBufferDataSize > 0) this->iomux->SetExec(this, nullptr);
            else this->DoSocketRecv(buffer, this->bytesToRead < size ? this->bytesToRead : size, this->recvPartial); // this->readUntilConnectionClose || this->recvPartial);
            break;

        case HTTP_CL_STATE_READ_CHUNK_SIZE:
            this->iomux->SetExec(this, nullptr);
            break;

        default:
            debug->printf("HTTPClient::Recv(): Unexpected state %u", this->state);
            ASSERT(false, "Assert because of unexpected state in HTTPClient::Recv()");
        }
    }
}


void HTTPClient::SocketSend(const void * buf, size_t len)
{
    this->execCnt++;
    this->socket->Send(buf, len);
}


void HTTPClient::DoSend(const byte * data, size_t size)
{
    bool hasToAuthenticate = this->authUserName != nullptr;
    bool preserveData = false;
    bool last = this->dataLast;
    this->chunkedTransfer = this->chunkedTransferCalled;    // reset the chunked transfer flag to the original flag of SetRequestType
    if (this->state == HTTP_CL_STATE_READY || this->state == HTTP_CL_STATE_AUTH_NTLM_T1M) {
        if (!hasToAuthenticate) {
            this->bytesToSend = this->chunkedTransfer ? size : this->contentLength;
        }

        CustomField * curField = this->customFieldList;
        while (curField) {
            this->httpHeader->SetCustomField(curField->fieldWithValue);
            curField = curField->next;
        }

        if (this->digest.nonce) SetDigestHeader();

        if (this->requestHasDataToSend) {
            if (!this->ntlmAuthenticated && !this->fakingAuth && (this->state == HTTP_CL_STATE_AUTH_NTLM_T1M || this->state == HTTP_CL_STATE_READY) && this->authUserName && *this->authUserName) {
                // now here we have a hack! If we know we may need authentication and we do a PUT/POST
                // with a chunked transfer or with a known contentLength where the App sends the data in chunks
                // we FAKE a first request with a content length of 0 to get the needed stuff for authentication
                // afterwards we'll send the real data
                last = true;
                preserveData = true;    // don't send out data now, but later
                this->fakingAuth = !this->sendContentForAuth;
                this->httpHeader->SetContentSize(this->sendContentForAuth ? size : 0);
            }
            else {
                this->fakingAuth = false;
                if (this->chunkedTransfer) { // first NTLM auth state doesn't require data
                    this->httpHeader->SetCustomField("Transfer-Encoding: chunked");
                }
                else {
                    this->httpHeader->SetContentSize(this->contentLength);
                }
            }
        }
        if (this->state != HTTP_CL_STATE_AUTH_NTLM_T1M) {
            this->state = HTTP_CL_STATE_HEADER_SENT;
        }

		this->DebugPrintHTTPHeader("DoSend(): HTTP header sent:");
		SocketSend(this->httpHeader->GetHeaderData(), this->httpHeader->GetHeaderSize());
    }

	if (this->requestHasDataToSend) {
        if (data != this->dataToSend) {
            if (data) {     // append data to dataToSend to support multiple Send calls without waiting for SendResults
                // The construct below is to make the Visual C++ analyser happy (ok, he wasn't that wrong - see C28182 and C6308).
                byte * tmp = (byte*)realloc(this->dataToSend, this->dataToSendSize + size);
                if (tmp == nullptr) {
                    ASSERT(false, "HTTPClient::DoSend() Realloc returned nullptr!");
                }
                else {
                    this->dataToSend = tmp;
                    memcpy(&this->dataToSend[this->dataToSendSize], data, size);
                }
                this->dataToSendSize += size;
            }
        }
        // if sendContentForAuth is set, we must send the data even in the first request without authentication
        if ((!preserveData || this->sendContentForAuth) && (this->state == HTTP_CL_STATE_HEADER_SENT || this->state == HTTP_CL_STATE_AUTH_NTLM_T1M)) {
            if (this->dataToSend && this->dataToSendSize) {
                if (this->chunkedTransfer) {    // send the data size as hex with a new line
                    char chunkHead[20];
                    int chunkHeadLen = _snprintf(chunkHead, sizeof(chunkHead), "%X\r\n", (unsigned)dataToSendSize);
                    this->SocketSend(chunkHead, chunkHeadLen);
                }
                this->SocketSend(this->dataToSend, this->dataToSendSize);
                if (this->chunkedTransfer) {    // send the new line to terminate the string
                    this->SocketSend("\r\n", 2);
                }
                else {
                    ASSERT(this->bytesToSend && this->bytesToSend >= this->dataToSendSize, "HTTPClient::Send(): you sent too much data which doesn't fit your contentLength given in SetRequestType");
                    // if we don't have to preserve the data for the retry auth request, we can decrease our send size
                    if (!preserveData) {
                        this->bytesToSend -= this->dataToSendSize;
                        if (!this->bytesToSend) last = true;
                    }
                }
                // if we don't have to preserve the data for the retry auth request, we can free the buffer
                if (!preserveData) {
                    free(this->dataToSend);
                    this->dataToSend = nullptr;
                    this->dataToSendSize = 0;
                }
            }
            if (this->chunkedTransfer && last) {
                this->SocketSend("0\r\n\r\n", 5);
            }
        }
    }
    else {
        last = true;
    }

    if (last) { // after having sent the last chunk, we want to read the header
        if (this->state != HTTP_CL_STATE_AUTH_NTLM_T1M) {
            this->state = HTTP_CL_STATE_READ_HEADER;
        }
    }
}


void HTTPClient::PassErrorToUser(http_error_t err)
{
    this->passErrorToUserFlags |= err;
}


void HTTPClient::Send(const byte * data, size_t size, bool last)
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::Send size:%i last:%i", this, this->user, size, last);
    this->log->HexDump(LOG_HTTP_CLIENT, data, size);

    // do not start a timer on send, because a SocketSendResult will be always triggert or causes a SocketShutdown
    this->sendCnt++;
    this->dataLast = last;
    this->DoSend(data, size);
}


void HTTPClient::DoShutdown()
{
    if (this->state != HTTP_CL_STATE_SHUTDOWN) {
        this->StopTimer();
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::Shutdown", this, this->user);
        if (this->socket) {
            this->state = HTTP_CL_STATE_SHUTDOWN;
            this->socket->Shutdown();
        }
        else if (this->state != HTTP_CL_STATE_RESOLVE_HOST_NAME) {
            // Socket is only nullptr, if the DNS Lookup failed. In that case we are already
            // in some kind of asynchronous state...
            if (this->consumingData) {
                free(this->userBuffer);
                this->userBuffer = this->savedUserBuffer;
                this->userBufferSize = this->savedUserBufferSize;
                this->state = this->savedState;
                this->lastResult = this->savedLastResult;
                this->consumingData = false;
            }
            if (userBuffer) {
                this->UserHTTPClientRecvCanceled(this->userBuffer);
                this->userBuffer = nullptr;
                this->userBufferSize = 0;
                this->userBufferAlreadyRead = 0;
            }
            this->state = HTTP_CL_STATE_SHUTDOWN;
            this->UserHTTPClientShutdown(this->shutdownReason);
        }
        else {
            this->state = HTTP_CL_STATE_SHUTDOWN;
        }
    }
}

void HTTPClient::Shutdown()
{
    // Make sure the shutdown won't end up in a redirect in case the HTTP client is going to reconnect on his own to another server.
    if (this->httpRedirectData != nullptr) {
        this->httpRedirectData->Cancel();
    }

    this->DoShutdown();
}

void HTTPClient::Reconnect()
{
    if (this->socket != nullptr) {
        this->state = HTTP_CL_STATE_RECONNECT;
        this->socket->Shutdown();
    }
    else this->ConnectSocket();
}


void HTTPClient::SetRequest()
{
	http_request_type_info ri = RequstTypeToRequestInfo(this->requestType);
	std::string methodHeader = std::string("Method: ") + ri.methodStr;
	
    this->httpHeader->SetRequest(ri.method, resource);
    this->httpHeader->SetCustomField("Host", hostName);
    this->httpHeader->SetCustomField("Accept: */*");
    this->httpHeader->SetCustomField("Connection: Keep-Alive");
    if (this->contentType && strlen(this->contentType) > 0 ) {
        this->httpHeader->SetContentType(contentType);
    }
	this->httpHeader->SetCustomField(methodHeader.c_str());
}


void HTTPClient::SetRequestType(http_request_type_t reqType, const char * resourceName, size_t contentLength, const char * contentType)
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::SetRequestType reqType:%i resourceName:%s contentLength:%u", this, this->user, reqType, resourceName, contentLength);
    //ASSERT(state == HTTP_CL_STATE_READY, "HTTPClient::SetRequestType called while state while still processing a previous request");
    free(this->resource);
    free(this->contentType);
    free(this->dataToSend);
    this->retryAuthenticated = false;   // reset authentication retry for each new request
    this->dataToSend = nullptr;
    this->dataToSendSize = 0;
    this->answerContentLength = 0;
    this->responseCode = 0;
    free(this->responseCodeDescription);
    this->responseCodeDescription = nullptr;
    this->contentType = _strdup(contentType);
    this->requestType = reqType;
    this->dataLast = false;
    this->lastResult = HTTP_RESULT_OK;
    this->state = HTTP_CL_STATE_READY;
    this->sendCnt = 0;

    class uri_dissector resourceParts;

    if (resourceParts.dissect_uri((char *)resourceName) ) {
        unsigned length = resourceName ? strlen(resourceName) + 1 : 1;
        this->resource = (char *)malloc(length);
        resourceParts.compose(this->resource, length, COMPOSE_PATH | COMPOSE_FILE | COMPOSE_ANCHOR | COMPOSE_QUERY | COMPOSE_PARAMS);
    }
    else {
        this->resource = _strdup(resourceName);
    }

    if (contentLength == HTTP_CLIENT_CHUNKED_TRANSFER) {
        this->chunkedTransfer = this->chunkedTransferCalled = true;
        this->contentLength = this->bytesToSend = 0;
    }
    else {
        this->chunkedTransfer = this->chunkedTransferCalled = false;
        this->contentLength = this->bytesToSend = contentLength;
    }
    this->requestHasDataToSend = contentLength > 0;

    this->SetRequest();
}

void HTTPClient::SetCustomHeaderField(const char * field, const char * value)
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::SetCustomHeaderField field:%s value:%s", this, this->user, field, value);
    this->customFieldList = new CustomField(field, value, this->customFieldList);
}

http_result_t HTTPClient::GetHTTPResult()
{
    log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::GetHTTPResult lastResult:%i", this, this->user, this->lastResult);
    return this->lastResult;
}

int HTTPClient::GetResponseCode()
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::GetResponseCode code:%i", this, this->user, this->responseCode);
    return this->responseCode;
}

const char * HTTPClient::GetResponseCodeDescription()
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::GetResponseCodeDescription desc:%s", this, this->user, this->responseCodeDescription);
    return this->responseCodeDescription;
}

size_t HTTPClient::GetHeaderFieldValueCount(const char * headerField)
{
    if (this->httpParser != nullptr) {
        const HTTPHeaderField * hf = this->httpParser->GetHeaderField(headerField);
        if (hf != nullptr) return hf->GetValueCount();
    }

    return 0;
}


const char * HTTPClient::GetHeaderFieldValue(const char * headerField, size_t index)
{
    if (this->httpParser != nullptr) {
        const HTTPHeaderField * hf = this->httpParser->GetHeaderField(headerField);
        if (hf == nullptr || index > hf->GetValueCount()) return nullptr;
        return hf->GetValue(index);
    }
    return nullptr;
}


size_t HTTPClient::GetContentLength(bool & chunked)
{
    if (this->answerContentLength == HTTP_CLIENT_CHUNKED_TRANSFER) {
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::GetContentLength contentLength:%u chunked:%s", this, this->user, 0, "TRUE");
        chunked = true;
        return 0;
    }
    else {
        log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::GetContentLength contentLength:%u chunked:%s readUntilConnectionClose:", this, this->user, this->answerContentLength, "FALSE", this->readUntilConnectionClose ? "FALSE" : "TRUE");
        chunked = false;
        return this->readUntilConnectionClose ? 0 : this->answerContentLength;
    }
}


int HTTPClient::GetSupportedAuthMethodsFromHeader()
{
    const HTTPHeaderField * field = this->httpParser->GetHeaderField("WWW-Authenticate");
    int authMeth = 0;

    if (field) {
        for (size_t i = 0; i < field->GetValueCount(); i++) {
            if (str::n_casecmp(field->GetValue(i), "NTLM", 4) == 0) authMeth |= HTTP_AUTH_NTLM;
            else if (str::n_casecmp(field->GetValue(i), "Basic", 5) == 0) authMeth |= HTTP_AUTH_BASIC;
            else if (str::n_casecmp(field->GetValue(i), "Digest", 6) == 0) authMeth |= HTTP_AUTH_DIGEST;
        }
    }

    return authMeth;
}


char * HTTPClient::GetNTLMType1Message()
{
    NTLMType1Message t1msg;
    t1msg.SetFlag(NTLM_NEGOTIATE_OEM);
    t1msg.SetFlag(NTLM_NEGOTIATE_UNICODE);
    t1msg.SetFlag(NTLM_NEGOTIATE_NTLM);
    t1msg.SetFlag(NTLM_NEGOTIATE_NTLM2);
    t1msg.SetFlag(NTLM_NEGOTIATE_ALWAYS_SIGN);

    size_t b64len = (((4 * t1msg.GetByteStreamLength() / 3) + 3) & ~3);
    char * result = (char *)malloc(b64len + 6);
    if (result != nullptr) {
        encode_base64(t1msg.GetByteStream(), &result[5], t1msg.GetByteStreamLength());
        memcpy(result, "NTLM ", 5);
    }
    else {
        ASSERT(false, "HTTPClient::GetNTLMType1Message(): malloc returned nullptr!");
    }
    return result;
}


char * HTTPClient::GetNTLMType3Message()
{
    // First we need the data send form the server...
    const HTTPHeaderField * field = this->httpParser->GetHeaderField("WWW-Authenticate");
    const char * tmp = nullptr;
    for (size_t i = 0; i < field->GetValueCount(); i++) {
        if (strncmp(field->GetValue(i), "NTLM", 4) == 0) {
            tmp = field->GetValue(i);
            break;
        }
    }

    if (tmp == nullptr || *(tmp + 4) == '\0' ) {
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::GetNTLMType3Message didn't receive Type 2 message", this, user);
        this->shutdownReason = HTTP_AUTHENTICATION_FAILED;
        this->Shutdown();
        return nullptr;
    }

	char * result = nullptr;
	try {
		tmp += 5; // Ignore "NTLM "
		NTLMType2Message type2Msg((byte *)tmp);
		NTLMType3Message type3Msg(this->authUserName, this->authUserPassword, nullptr, nullptr, &type2Msg);
		//type3Msg.SetFlag(NTLM_NEGOTIATE_OEM);
		type3Msg.SetFlag(NTLM_REQUEST_TARGET);
	
		size_t t3msglen = type3Msg.GetByteStreamLength();
		size_t b64len = (((4 * t3msglen / 3) + 3) & ~3);
		result = (char *)malloc(b64len + 6);
		encode_base64(type3Msg.GetByteStream(), &result[5], t3msglen);
		memcpy(result, "NTLM ", 5);
	}
	catch (std::runtime_error & e) {
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::GetNTLMType3Message failed: %s", this, user, e.what());
	}
	
    return result;
}


void HTTPClient::AuthenticateBasic()
{
    char tmp[1024];
    char tmp2[1024];
    int lenTmp = _snprintf(tmp, sizeof(tmp), "Basic ");
    int len = _snprintf(tmp2, sizeof(tmp2), "%s:%s", this->authUserName, this->authUserPassword);
    encode_base64((byte *)tmp2, &tmp[lenTmp], len);
    this->SetRequest();
    this->httpHeader->SetCustomField("Authorization", tmp);
    this->state = HTTP_CL_STATE_READY;
    this->DoSend(this->dataToSend, this->dataToSendSize);
}


void HTTPClient::AuthenticateDigest()
{
    const char * tmp = nullptr;
    const HTTPHeaderField * field = this->httpParser->GetHeaderField("WWW-Authenticate");
    if (field) {
        for (size_t i = 0; i < field->GetValueCount(); i++) {
            if (strncmp(field->GetValue(i), "Digest", 6) == 0) {
                tmp = (char *)field->GetValue(i);
                tmp += 7;   // ignore "Digest "
                break;
            }
        }
    }
    char * copy = _strdup(tmp);
    char * next = copy;
    const char * realm = "";
    const char * qop = nullptr;
    const char * nonce = "";
    const char * opaque = nullptr;
    const char * algorithm = "";
    bool stale = false;
    bool cont = true;

    while (*next == ' ') ++next;

    while (cont) {
        char * p = strchr(next, '=');
        if (!p) cont = false;
        else {
            char * keyWord = next;
            char * value = p + 1;
            *p++ = '\0';
            char * e;
            if (*p == '\"') {
                ++p;
                e = strchr(p, '\"');
            }
            else e = strchr(p, ',');

            if (e) {
                *e = '\0';
                next = e + 1;
                while (*next == ' ' || *next == ',') ++next;
            }

            // Remove leading and ending ", if there is any...
            if (*value == '\"') value++;
            size_t tmp = strlen(value);
            if (value[tmp - 1] == '\"') value[tmp - 1] = '\0';

            if (strcmp(keyWord, "realm") == 0) realm = value;
            else if (strcmp(keyWord, "nonce") == 0) nonce = value;
            else if (strcmp(keyWord, "qop") == 0) qop = value;
            else if (strcmp(keyWord, "opaque") == 0) opaque = value;
            else if (strcmp(keyWord, "algorithm") == 0) algorithm = value;
            else if (strcmp(keyWord, "stale") == 0) {
                if (!strcasecmp(value, "true")) {
                    stale = true;
                }
            }
        }
    }
    SetDigest(nonce, opaque, qop, realm, algorithm, stale);
    free(copy);
    this->state = HTTP_CL_STATE_READY;
    // the digest header will be appended automatically inside Send, as we now have digest values
    this->SetRequest();
    this->DoSend(this->dataToSend, this->dataToSendSize);
}


void HTTPClient::MakeMD5String(const byte * in, size_t len, char * out33byte)
{
    byte buf[HASH_SIZE_MD5];
    hash md5;
    md5.init(HASH_MD5);
    md5.update(in, len);
    md5.final(buf);
    str::from_hexmem(buf, sizeof(buf), out33byte);
}


void HTTPClient::SetDigestHeader()
{
    char cnonce[17];
    char nc[9];
    char ha1[HASH_SIZE_MD5 * 2 + 1];
    char ha2[HASH_SIZE_MD5 * 2 + 1];
    char response[HASH_SIZE_MD5 * 2 + 1];
    size_t bufSize = GetDigestAuthBufferSize();
    char * buf = static_cast<char *>(malloc(bufSize));
	http_request_type_info ri = RequstTypeToRequestInfo(this->requestType);
	
    this->digest.nc++;    // increment nc for every new request
    _snprintf(cnonce, sizeof(cnonce), "%08x%08x", IRandom::GetRandom(), IRandom::GetRandom());
    _snprintf(nc, sizeof(nc), "%08x", this->digest.nc);
    
    // ha1 = MD5(username:realm:password)
    int len = _snprintf(buf, bufSize, "%s:%s:%s", this->authUserName, this->digest.realm, this->authUserPassword);
    MakeMD5String((byte *)buf, len, ha1);
    if (this->digest.algorithm && !strcasecmp(this->digest.algorithm, "MD5-sess")) {
        // ha1 = MD5(username:realm:password):nonce:cnonce
        len = _snprintf(buf, bufSize, "%s:%s:%s", ha1, this->digest.nonce, cnonce);
        MakeMD5String((byte *)buf, len, ha1);
    }

    // ha2 = MD5(method:uri)
    len = _snprintf(buf, bufSize, "%s:%s", ri.methodStr.c_str(), this->resource);
    MakeMD5String((byte *)buf, len, ha2);

    // 1. response digest with qop    MD5(ha1:nonce:nc:cnonce:qop:h2)
    // 2. response digest without qop MD5(ha1:nonce:h2)
    len = _snprintf(buf, bufSize, "%s:%s:", ha1, this->digest.nonce);
    if (this->digest.qop && *this->digest.qop) {
        len += _snprintf(&buf[len], bufSize - len, "%s:%s:%s:", nc, cnonce, this->digest.qop);
    }
    len += _snprintf(&buf[len], bufSize - len, "%s", ha2);
    MakeMD5String((byte *)buf, len, response);
    
    len = _snprintf(buf, bufSize, "Digest username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", algorithm=%s, cnonce=\"%s\", response=\"%s\"",
        this->authUserName, this->digest.realm, this->digest.nonce, this->resource, this->digest.algorithm, cnonce, response);
    if (this->digest.opaque) {
        len += _snprintf(&buf[len], bufSize - len, ", opaque=\"%s\"", digest.opaque);
    }
    if (this->digest.qop) {
        len += _snprintf(&buf[len], bufSize - len, ", qop=auth, nc=%s", nc);
    }
    this->log->Log(LOG_HTTP_CLIENT, "%s method:%s", buf, ri.methodStr.c_str());
    this->httpHeader->SetCustomField("Authorization", buf);
    free(buf);
}


size_t HTTPClient::GetDigestAuthBufferSize()
{
    // Well - we can use a buffer of 10000 bytes or so what. But on the other hand, calculate that stuff will be more accurate.
    // Because with that, we're on the save side that whatever strings had been send for realm and nonce and how long the
    // requested resource string will be - it will be enough memory to store all the data!

    size_t realmLen = strlen(this->digest.realm);
    size_t bufSize = 80; // base for the static strings added to the response output.
    bufSize += realmLen;
    bufSize += strlen(this->digest.nonce);
    bufSize += strlen(this->resource);
    bufSize += strlen(this->authUserName);
    bufSize += 16; // 2x 8 chars for the cnonce value.
    bufSize += (HASH_SIZE_MD5 * 2); // The length of the MD5Sum for the response value

    // The values for the authentication header field must be seperated by comma with an optional space char. However, it seems
    // that the space char after the comma isn't as optional as I thought (some webservers have a problem if the space is
    // missing). So it will be added which means, the buffer size must be adjusted, too. The first 6 spaces are fixed, because
    // there are at least 7 values seperated by comma (= 6 commas). However, if an opaque value exists, one comma and with that
    // one space will be added, if qop is set, the nonce counter must be added as well as the qop option which makes two additional
    // commas with two additional spaces.
    bufSize += 6;
    
    if (this->digest.opaque) bufSize += 10 + strlen(this->digest.opaque) + 1;
    if (this->digest.qop) bufSize += 21 + strlen(this->digest.qop) + 2; // ", qop=auth, nc=" + the nc string, wich is 8 chars long

    // Because the buffer will also be used for temporary operations, we must make sure that it is big enough.
    size_t tmpSize = strlen(this->authUserName) + strlen(this->authUserPassword) + realmLen + 2;
    if (tmpSize > bufSize) bufSize += (tmpSize - bufSize);

    if (this->digest.algorithm && *this->digest.algorithm != '\0') {
        size_t algLen = strlen(this->digest.algorithm);
        bufSize += algLen;
        tmpSize = algLen + 37; // 38 = 21 for qop + 16 for cnonce
        if (tmpSize > bufSize) bufSize += tmpSize - bufSize;
    }

    return bufSize + 1; // + 1 to include the closing zero.
}


void HTTPClient::SetDigest(const char * nonce, const char * opaque, const char * qop, const char * realm, const char * algorithm, bool stale)
{
    free(this->digest.nonce);
    free(this->digest.opaque);
    free(this->digest.qop);
    free(this->digest.realm);
    free(this->digest.algorithm);
    this->digest.nc = 0;
    this->digest.nonce = (nonce ? _strdup(nonce) : nullptr);
    this->digest.opaque = (opaque ? _strdup(opaque) : nullptr);
    this->digest.qop = (qop ? _strdup(qop) : nullptr);
    this->digest.realm = (realm  ? _strdup(realm) : nullptr);
    this->digest.algorithm = (algorithm ? _strdup(algorithm) : nullptr);
}


void HTTPClient::AuthenticateNTLM()
{
    this->ntlmAuthenticated = false;

    this->SetRequest();
    char * tmp;

    switch (this->state)
    {
    case HTTP_CL_STATE_REAUTHENTICATE:
	    tmp = this->GetNTLMType1Message();
	    this->httpHeader->SetCustomField("Authorization", tmp);
	    free(tmp);
        this->fakingAuth = false; // we need to reset fakingAuth in order to use the fakingAuth mechanism again for the next Send
        this->state = HTTP_CL_STATE_AUTH_NTLM_T1M;
        // dataToSend & dataToSendSize will be set in HTTPClient::Send(). Because make a backup
        // of that data and restore it after the Send() call...
        this->DoSend(this->dataToSend, this->dataToSendSize);
        this->retryAuthenticated = false;   // we have a two step authentication here
        this->DoSocketRecv(this->headerBuffer, sizeof(this->headerBuffer), true);
        break;

    case HTTP_CL_STATE_AUTH_NTLM_T1M:
        tmp = this->GetNTLMType3Message();
	    if (tmp == nullptr) {
		    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient::AuthenticateNTLM(): Critical error while authenticating via NTLM.");
		    this->shutdownReason = HTTP_FAILURE;
		    this->Shutdown();
	    }
	    else {
            this->httpHeader->SetCustomField("Authorization", tmp);
            free(tmp);
            this->ntlmAuthenticated = true;
            this->state = HTTP_CL_STATE_READY;
            this->DoSend(this->dataToSend, this->dataToSendSize);
	    }
        free(this->dataToSend);
        this->dataToSend = nullptr;
        this->dataToSendSize = 0;
        break;

    default:
        break;
    }
}


void HTTPClient::StartConsumeData(size_t len, size_t dataRead, http_result_t res)
{
    this->savedState = this->state;
    this->savedLastResult = this->lastResult;
    this->savedUserBuffer = this->userBuffer;
    this->savedUserBufferSize = this->userBufferSize;
    this->userBuffer = (byte *)malloc(1024);
    this->userBufferSize = 1024;
    this->lastResult = res;
    this->headerBufferDataSize = len - dataRead;
    this->headerBufferDataStart = headerBuffer + dataRead;
    this->state = HTTP_CL_STATE_START_READ_CONTENT;
    this->consumingData = true;
    this->DoRecv();
}


void HTTPClient::PrepareRedirect(size_t len, size_t dataRead, bool isPermanent)
{
    const HTTPHeaderField * field = this->httpParser->GetHeaderField("Location");
    if (!field) {
        debug->printf("HTTPClient HTTP 302 received without location field");
        this->shutdownReason = HTTP_FAILURE;
        this->Shutdown();
        return;
    }

    const char * location = field->GetValue(0);
    if (location == nullptr || *location == '\0') {
        debug->printf("HTTPClient HTTP 302 received, but headerfield is empty");
        this->shutdownReason = HTTP_FAILURE;
        this->Shutdown();
        return;
    }

    this->httpRedirectData = new HTTPRedirectData(isPermanent);
    this->httpRedirectData->RequestData(this->requestType, this->contentType, this->contentLength);
    this->httpRedirectData->AuthData(this->authUserName, this->authUserPassword, this->authMethods);
    this->httpRedirectData->CustomHeaderFields(this->customFieldList);
    this->httpRedirectData->UserBuffer(this->userBuffer, this->userBufferSize, this->recvPartial);

    this->customFieldList = nullptr;
    this->userBuffer = nullptr;
    this->userBufferSize = 0;
    this->recvPartial = false;
    
    // There are three types of redirect:
    //   - relative-absolut (same server, just different path - identified by the / at the begin of lcation)
    //   - relative (same server, path changed to a given sub-path of the request - identified by not having a / and no tld at the begining)
    //   - absolut (complete new address, uncluding new server - identified by a tld at the beginning.
    //
    // The first two are easy to handle, the last one need a reset of the HTTP client to reconnect to the new URL.

    uri_dissector uriDisp;
    if (uriDisp.dissect_uri((char *)location) && uriDisp.fqdn != nullptr) { // absolute redirect
        this->httpRedirectData->Type(HTTPRedirectData::redirect_type_t::ABSOLUTE);
        if (uriDisp.path == nullptr) this->httpRedirectData->Location(location, "/");
        else this->httpRedirectData->Location({ location, size_t(uriDisp.path - location) }, uriDisp.path);
    }
    else if (*location == '/') { // relative aboslute redirect
        this->httpRedirectData->Type(HTTPRedirectData::redirect_type_t::RELATIVE_ABSOLUTE);
        this->httpRedirectData->Location(this->hostName, location);
    }
    else { // relative redirect
        std::string newLoc= this->resource;
        size_t pos = newLoc.rfind('/');
        if (pos != std::string::npos) {
            newLoc.erase(pos);
        }
        newLoc += std::string("/") + location;

        this->httpRedirectData->Type(HTTPRedirectData::redirect_type_t::RELATIVE);
        this->httpRedirectData->Location(this->hostName, newLoc.c_str());
    }

    free(this->resource);
    this->resource = nullptr;
    this->StartConsumeData(len, dataRead);
}


void HTTPClient::FollowRedirect(bool reconnectComplete)
{
    if (this->httpRedirectData->RedirectCanceled()) {
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient::FollowRedirect(%08x): Redirect in progress had been canceld due HTTP client shutdown request.", this);
        delete this->httpRedirectData;
        this->httpRedirectData = nullptr;
    }
    else if (this->httpRedirectData->Type() == HTTPRedirectData::redirect_type_t::ABSOLUTE && !reconnectComplete) {
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient::FollowRedirect(%08x): Shutting down HTTP client to reconnect to new server.", this);
        this->DoShutdown();
    }
    else {
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient::FollowRedirect(%08x): Restarting request to follow redirect.", this);

        this->SetRequestType(this->httpRedirectData->RequestMethod(), this->httpRedirectData->Resource(), this->httpRedirectData->ContentLength(), this->httpRedirectData->ContentType());
        if (this->httpRedirectData->HasCustomFields()) {
            this->customFieldList = this->httpRedirectData->CustomHeaderFields();
            this->httpRedirectData->CustomHeaderFields(nullptr);
        }

        this->userBuffer = this->httpRedirectData->UserBuffer();
        this->userBufferSize = this->httpRedirectData->UserBufferSize();
        this->recvPartial = this->httpRedirectData->RecvPartitial();
        this->sendCnt = 0; // Reset the send counter, since we need to start over.
        this->state = HTTP_CL_STATE_READY;
        this->user->HTTPClientRedirectComplete(this, this->httpRedirectData->Server(), this->httpRedirectData->Resource(), this->httpRedirectData->IsPermanent());
        delete this->httpRedirectData;
        this->httpRedirectData = nullptr;
    }
}

void HTTPClient::RecvAuthData(byte * buffer, size_t len)
{
    this->authToRead -= len;
    if (!this->authToRead) Authenticate();
    else this->DoSocketRecv(this->headerBuffer, sizeof(this->headerBuffer), true);
}


void HTTPClient::Authenticate()
{
    if (this->state != HTTP_CL_STATE_AUTH_NTLM_T1M) this->state = HTTP_CL_STATE_REAUTHENTICATE;

    int authMeth;
    if (this->retryAuthenticated || !this->authUserName || !this->authUserPassword) {    // we can cancel the authentication at once if no username/password is set or if we already tried to authenticate once
        shutdownReason = HTTP_AUTHENTICATION_FAILED;
        this->Shutdown();
        return;
    }
    this->retryAuthenticated = true;
    switch (state)
    {
    case HTTP_CL_STATE_REAUTHENTICATE:
        authMeth = this->GetSupportedAuthMethodsFromHeader();
        if ((authMeth & HTTP_AUTH_NTLM) != 0 && (authMethods & HTTP_AUTH_NTLM) != 0) {
            this->AuthenticateNTLM();
        }
        else if ((authMeth & HTTP_AUTH_DIGEST) != 0 && (authMethods & HTTP_AUTH_DIGEST) != 0) {
            this->AuthenticateDigest();
        }
        else if ((authMeth & HTTP_AUTH_BASIC) != 0 && (authMethods & HTTP_AUTH_BASIC) != 0) {
            this->AuthenticateBasic();
        }
        else {
            this->shutdownReason = HTTP_AUTHENTICATION_FAILED;
            this->Shutdown();
        }
        break;

    case HTTP_CL_STATE_AUTH_NTLM_T1M:
        this->AuthenticateNTLM();
        break;

    default:
        break;
    }
}


void HTTPClient::StartReadRequestData()
{
    if (this->chunkedTransfer) this->StartReadChunk();
    else if (this->answerContentLength > 0 || this->readUntilConnectionClose) {
        this->state = HTTP_CL_STATE_READ_CONTENT;
        if (this->userBuffer && this->headerBufferDataStart) this->CopyDataAlreadyLoaded();
    }
    else this->UserHTTPClientRecvResult(true);
}


void HTTPClient::StartReadChunk()
{
    size_t numStart = strlen(this->chunkSizeNum);
    size_t numLen = sizeof(this->chunkSizeNum) - 1 - numStart;
    if (numLen > this->headerBufferDataSize) {
        numLen = this->headerBufferDataSize;
    }
    memcpy(&this->chunkSizeNum[numStart], this->headerBufferDataStart, numLen);
    this->chunkSizeNum[numStart + numLen] = '\0';

    char * startOfNum = &this->chunkSizeNum[((this->chunkSizeNum[0] == '\r') && (this->chunkSizeNum[1] == '\n')) ? 2 : 0];
    char * endOfNum = strstr(startOfNum, CLRF);
    if (!endOfNum) {
        if (numStart + numLen >= sizeof(chunkSizeNum) - 1) {
            debug->printf("HTTPClient::StartReadChunk(): failed to read chunk size %s %s", this->resource, this->chunkSizeNum);
            this->shutdownReason = HTTP_BYTE_STREAM_BROKEN;
            this->Shutdown();
            return;
        }
        this->headerBufferDataSize = 0;
        this->state = HTTP_CL_STATE_READ_CHUNK_SIZE;
        this->DoSocketRecv(this->headerBuffer, sizeof(this->chunkSizeNum) - 1 - (numStart + numLen), true);
        return;
    }

    numLen = endOfNum - &this->chunkSizeNum[numStart];
    this->chunkSizeNum[numStart + numLen] = '\0';

    numLen += 2; // remove also CL RF
    this->headerBufferDataStart += numLen;
    this->headerBufferDataSize -= numLen;
    endOfNum = startOfNum;
    this->bytesToRead = strtol(startOfNum, &endOfNum, 16);
    this->chunkSizeNum[0] = '\0';

    if (this->bytesToRead == 0) {
        if ((endOfNum == startOfNum)
         || ((this->headerBufferDataSize > 0) && (this->headerBufferDataStart[0] != '\r'))
         || ((this->headerBufferDataSize > 1) && (this->headerBufferDataStart[1] != '\n'))
         || (this->headerBufferDataSize > 2)) {
            debug->printf("HTTPClient::StartReadChunk(): Unexpected data after end of chunks");
            this->shutdownReason = HTTP_BYTE_STREAM_BROKEN;
            this->Shutdown();
            return;
        }

        if (headerBufferDataSize < 2) {
            this->state = HTTP_CL_STATE_READ_CHUNK_END;
            this->DoSocketRecv(headerBuffer, 2 - headerBufferDataSize);
            return;
        }

        this->UserHTTPClientRecvResult(true);
        return;
    }

    this->state = HTTP_CL_STATE_READ_CHUNKED;
    this->CopyDataAlreadyLoaded();
}


void HTTPClient::CopyDataAlreadyLoaded()
{
    size_t userBufferRest = this->userBufferSize - this->userBufferAlreadyRead;

    if (userBufferRest == 0) {
        this->UserHTTPClientRecvResult(false);
        return;
    }
    
    size_t dataToRead = this->bytesToRead > userBufferRest ? userBufferRest : this->bytesToRead;
    if (dataToRead > this->headerBufferDataSize) dataToRead = this->headerBufferDataSize;
    if (dataToRead > 0) {
        memcpy(this->userBuffer + this->userBufferAlreadyRead, this->headerBufferDataStart, dataToRead);

        this->AdjustBufferCounter(dataToRead);

        userBufferRest -= dataToRead;
        this->headerBufferDataSize -= dataToRead;
        this->headerBufferDataStart += dataToRead;
    }

    if (this->bytesToRead == 0) {
        if (this->state == HTTP_CL_STATE_READ_CHUNKED) {
            this->state = HTTP_CL_STATE_READ_CHUNK_SIZE;
            if (this->headerBufferDataSize > 0) this->StartReadChunk();
            else this->DoSocketRecv(this->headerBuffer, sizeof(this->chunkSizeNum) - 1, true);
        }
        else {
            this->UserHTTPClientRecvResult(true);
        }
    }
    else {
        // Check if dataToRead is zero which can happen in case of chunk encoded reading and a whole chunk number had been sent.
        // If not checked, a user recv result with len 0 will be triggered - and that shouldn't be the case here.
        if (userBufferRest > 0 && (!this->recvPartial || dataToRead == 0)) {
            this->DoSocketRecv(this->userBuffer + this->userBufferAlreadyRead, userBufferRest < this->bytesToRead ? userBufferRest : this->bytesToRead, this->readUntilConnectionClose || this->recvPartial);
        }
        else {
            this->UserHTTPClientRecvResult(false);
        }
    }
}


void HTTPClient::SocketConnectComplete(ISocket * const socket)
{
    this->StopTimer();
    this->timeoutOnConnect = false;

    this->connected = true;
    this->contentLength = 0;
    this->bytesToRead = 0;
    this->bytesToSend = 0;
    this->userBufferAlreadyRead = 0;
    memset(this->chunkSizeNum, 0, sizeof(this->chunkSizeNum));
    this->dataToSend = nullptr;
    this->dataToSendSize = 0;
    this->retryAuthenticated = false;
    this->fakingAuth = false;
    this->ntlmAuthenticated = false;
    this->SetDigest(nullptr, nullptr, nullptr, nullptr, nullptr, false);  // this frees the digest params

    this->chunkedTransfer = this->chunkedTransferCalled = false;
    this->lastResult = HTTP_RESULT_OK;
    this->responseCode = 0;
    free(this->responseCodeDescription);
    this->responseCodeDescription = nullptr;
    this->headerBuffer[0] = '\0';
    this->headerBufferDataStart = nullptr;
    this->headerBufferDataSize = 0;
    this->userBuffer = nullptr;
    this->userBufferSize = 0;

    this->execCnt = 0;
    delete this->httpParser;
    this->httpParser = new HTTPParser();

    if (this->state == HTTP_CL_STATE_CONNECT_TUNNEL) {
        char buf[1024];
        bool useIPv6 = strchr(this->hostName, ':') != nullptr;
        int len = snprintf(buf, sizeof(buf), useIPv6 ? "CONNECT [%s]:%u HTTP/1.1\r\nHost: [%s]:%u\r\n\r\n" : "CONNECT %s:%u HTTP/1.1\r\nHost: %s:%u\r\n\r\n", this->hostName, this->port, this->hostName, this->port);
        this->SocketSend(buf, len);
        this->DoSocketRecv(this->headerBuffer, sizeof(this->headerBuffer), true);
    }
    else {
        this->state = HTTP_CL_STATE_READY;
        if (this->httpRedirectData != nullptr) {
            this->log->Log(LOG_HTTP_CLIENT, "HTTPClient::SocketConnectComplete(%08x): Reconnect completed, ", this);
            this->FollowRedirect(true);
        }
        else {
            this->UserHTTPClientConnectComplete();
        }
    }
}


void HTTPClient::SocketUpgradeToTLSComplete(ISocket* const socket)
{
    SocketConnectComplete(socket);
}


void HTTPClient::SocketShutdown(ISocket * const socket, shutdownreason_t reason)
{
    this->StopTimer();
    bool wasTimeoutOnConnect = this->timeoutOnConnect;
    this->timeoutOnConnect = false;

    this->connected = false;

    if (this->shutdownReason == HTTP_SHUTDOWN_NORMAL) {
        switch (reason)
        {
        case SOCKET_SHUTDOWN_NORMAL:
            this->shutdownReason = HTTP_SHUTDOWN_NORMAL;
            break;

        case SOCKET_SHUTDOWN_BY_PEER:
            this->shutdownReason = HTTP_SHUTDOWN_NORMAL;
            break;

        case SOCKET_LOST:
            this->shutdownReason = HTTP_SOCKET_LOST;
            break;

        case SOCKET_ADDRESS_INVALID:
            this->shutdownReason = HTTP_ADDRESS_INVALID;
            break;

        case SOCKET_CONNECT_REJECTED:
            this->shutdownReason = HTTP_CONNECTION_REFUSED;
            break;

        case SOCKET_CONNECT_FAILED:
            this->shutdownReason = HTTP_CONNECT_FAILED;
            break;

        case SOCKET_OPEN_FAILED:
        case SOCKET_TLS_HANDSHAKE_FAILED:
            this->shutdownReason = HTTP_SOCKET_ERROR;
            break;

        default:
            debug->printf("HTTPClient::SocketShutdown(): recevied unexpected shutdownReason %u");
            break;
        }
    }
    
    delete this->socket;
    this->socket = nullptr;

    const char * host = this->proxy ? this->proxy->host : this->hostName;
    if (host
     && ((this->proxy ? this->proxyIPAddress : this->hostIPAddress)[0] != 0)
     && ((this->state == HTTP_CL_STATE_RESOLVED_HOST_NAME) || (this->state == HTTP_CL_STATE_CONNECT_TUNNEL) || wasTimeoutOnConnect)) {
        char * p = 0;
        str::to_ip(host, &p, /*prefix*/0);
        if ((p == host) && this->dns) {
            this->log->Log(LOG_HTTP_CLIENT, "HTTPClient::SocketShutdown(%08xm %08x): invalidating IP address %s (port %u) for host %s.", this, this->user, this->proxy ? this->proxyIPAddress : this->hostIPAddress, this->proxy ? ((this->proxy->port != 0) ? this->proxy->port : 80) : this->port, host);
            this->dns->UnsuitableAddress(host, this->log, this->proxy ? this->proxyIPAddress : this->hostIPAddress, this->proxy ? ((this->proxy->port != 0) ? this->proxy->port : 80) : this->port);
            if (!this->dnsRetried) { // just retry DNS resolution once, as we otherwise may run into an infinite loop, as the DNS module unmarks unsuitable addresses if no more addresses can be resolved
                // mark current address as unsuitable, as the IP address was not available at all and retry DNS resolution to see if there is another address
                // this helps e.g. if an IPv6 address is resolved first but can't be used (ARM gateway) so a second IPv4 resolved address may work then
                this->dnsRetried = true;
                this->ResolveDns();
                return;
            }
        }
        if (this->proxy) {
            this->proxy = this->proxy->goNext();
            ConnectProxy();
            return;
        }
    }

    // If we are not in shtudown state, and the reason won't prevent from connect again, try to connect again
    bool doReconnect = this->state == HTTP_CL_STATE_RECONNECT;
    this->state = HTTP_CL_STATE_OFFLINE;

    if (doReconnect) {
        this->ConnectSocket();
        return;
    }

    if (this->userBuffer) {
        // Temporarily set the state to shutdown, to prevent a potential shutdown call from the RecvResult callback
        // to be handled. Because if that isn't handled, the app  will crash...
        http_cl_state_t s = this->state;
        this->state = HTTP_CL_STATE_SHUTDOWN;
        if (this->readUntilConnectionClose) this->UserHTTPClientRecvResult(true);
        else UserHTTPClientRecvCanceled(this->userBuffer);
        this->state = s;

        this->userBuffer = nullptr;
        this->userBufferSize = 0;
        this->userBufferAlreadyRead = 0;
    }

    if (this->httpRedirectData != nullptr && !this->httpRedirectData->RedirectCanceled()) {
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient::SocketShutdown(%08x): HTTP client shutdown because of absolute redirect - reconnecting.", this);
        this->Connect(this->httpRedirectData->Server(), this->httpRedirectData->AuthUser(), this->httpRedirectData->AuthPwd(), this->httpRedirectData->AuthMethod());
    }
    else {
        this->UserHTTPClientShutdown(this->shutdownReason);
    }
}


void HTTPClient::SocketSendResult(ISocket * const socket)
{
    this->execCnt--;
    if (this->execCnt == 0) {
        switch (this->state) {
        case HTTP_CL_STATE_AUTH_NTLM_T1M:  // no send result here
            break;

        default:
            if (this->state == HTTP_CL_STATE_READ_HEADER) {
                // we now start reading the header to trigger HTTPClientRequestComplete
                this->DoSocketRecv(this->headerBuffer, sizeof(this->headerBuffer), true);
            }
            else {
                if (!this->fakingAuth && !this->consumingData) {
                    dword sendCntTmp = this->sendCnt;
                    // work with a copy from sendCnt, as sendCnt might be incremented again within UserHTTPClientSendResult
                    while (sendCntTmp != 0) { // trigger a send result for each previous Send call
                        this->UserHTTPClientSendResult();
                        --sendCntTmp;
                    }
                }
            }
            break;
        }
    }
}


void HTTPClient::SocketRecvResult(ISocket * const socket, void * buf, size_t len)
{
    this->StopTimer();

    //printf("---------------------------------------------------------------\n");
    //this->log->HexDump(LOG_HTTP_CLIENT, buf, len);
    //printf("\n%.*s\n", len, (char *) buf);
    //printf("---------------------------------------------------------------\n");

    switch (state)
    {
    case HTTP_CL_STATE_READ_CONTENT:
        this->AdjustBufferCounter(len);

        // We need some kind of special handling: if the mode to read until the connection closes is active, and no partial read had been
        // requested AND the user buffer is not jet full, make an other recv call instead of informing the user of a recv result.
        if (this->readUntilConnectionClose && !this->recvPartial && this->userBufferAlreadyRead < this->userBufferSize) {
            this->DoSocketRecv(this->userBuffer + this->userBufferAlreadyRead, this->userBufferSize - this->userBufferAlreadyRead, true);
            return;
        }

        this->UserHTTPClientRecvResult(this->bytesToRead == 0);
        break;

    case HTTP_CL_STATE_READ_CHUNKED:
        this->AdjustBufferCounter(len);
        
        if (this->userBufferAlreadyRead == this->userBufferSize || this->recvPartial) {
            if (this->bytesToRead == 0) this->state = HTTP_CL_STATE_READ_CHUNK_SIZE;
            this->UserHTTPClientRecvResult(false);
        }
        else {
            this->state = HTTP_CL_STATE_READ_CHUNK_SIZE;
            this->DoSocketRecv(headerBuffer, sizeof(chunkSizeNum) - 1, true);
        }
        return;

    case HTTP_CL_STATE_READ_CHUNK_SIZE:
        this->headerBufferDataStart = (byte *)buf;
        this->headerBufferDataSize = len;
        this->StartReadChunk();
        return;

    case HTTP_CL_STATE_READ_CHUNK_END:
        if ((((const char *)buf)[len-1] != '\n') || ((len > 1) && (((const char *)buf)[len-2] != '\r'))) {
            debug->printf("HTTPClient::SocketRecvResult(): Unexpected data after end of chunks");
            this->shutdownReason = HTTP_BYTE_STREAM_BROKEN;
            this->Shutdown();
            return;
        }
        this->UserHTTPClientRecvResult(true);
        break;

    case HTTP_CL_STATE_READ_AUTH_DATA:
        this->RecvAuthData((byte *)buf, len);
        break;

    default:
        this->ParseRequestAnswer((char *)buf, len);
    }
}


void HTTPClient::SocketRecvCanceled(ISocket * const socket, void * buf)
{
}


inline void HTTPClient::HandleRequestAsOK(size_t len, size_t dataRead, http_result_t res)
{
    this->lastResult = res;
    this->headerBufferDataSize = len - dataRead;
    this->headerBufferDataStart = headerBuffer + dataRead;
    this->state = HTTP_CL_STATE_START_READ_CONTENT;
    this->UserHTTPClientRequestComplete();
}


void HTTPClient::ParseRequestAnswer(char * buf, size_t len)
{
    //printf("%.*s\n--------------------------\n", len, buf);
    size_t dataRead = this->httpParser->ParseData((char *)buf, len);

    switch (this->httpParser->GetParseResult())
    {
    case HTTPParser::HTTP_PARSE_DONE:
        // nothing else then debug print to do here...
		this->DebugPrintHTTPHeader("ParseRequestAnswer(): HTTP header received:", true);
		break;

    case HTTPParser::HTTP_PARSE_ERROR:
        this->shutdownReason = HTTP_BYTE_STREAM_BROKEN;
        this->Shutdown();
        return;

    case HTTPParser::HTTP_PARSE_NEED_DATA:
        this->DoSocketRecv(headerBuffer, sizeof(headerBuffer), true);
        return;

    default:
        break;
    }

    if (this->state == HTTP_CL_STATE_CONNECT_TUNNEL) {
        if ((this->httpParser->GetHTTPResultCode()->code >= 200) && (this->httpParser->GetHTTPResultCode()->code < 300)) {
            this->state = HTTP_CL_STATE_RESOLVED_HOST_NAME;
            this->retryAuthenticated = false;
            if (this->useTLS) {
                this->socket = this->tlsSocketProvider->UpgradeToTLSSocket(this->socket, this->iomux, this, this->log, NULL);
            }
            else {
                this->SocketConnectComplete(this->socket);
            }
        }
        else if ((this->httpParser->GetHTTPResultCode()->code == 407) && !this->retryAuthenticated && proxy->usr && proxy->pwd) {
            this->retryAuthenticated = true;
            char tmp[1024];
            char tmp2[1024];
            int l = _sprintf(tmp, "%s:%s", proxy->usr, proxy->pwd);
            encode_base64((byte *)tmp2, tmp, l);
            bool useIPv6 = strchr(this->hostName, ':') != nullptr;
            l = snprintf(tmp, sizeof(tmp), useIPv6 ? "CONNECT [%s]:%u HTTP/1.1\r\nHost: [%s]:%u\r\nProxy-Authorization: Basic %s\r\n\r\n" :
                "CONNECT %s:%u HTTP/1.1\r\nHost: %s:%u\r\nProxy-Authorization: Basic %s\r\n\r\n", this->hostName, this->port, this->hostName, this->port, tmp2);
            this->SocketSend(tmp, l);
            this->DoSocketRecv(this->headerBuffer, sizeof(this->headerBuffer), true);
        }
        else {
            this->socket->Shutdown();
        }
        return;
    }

    this->chunkedTransfer = this->httpParser->IsChunkedTransfer();
    if (this->chunkedTransfer) {
        this->answerContentLength = HTTP_CLIENT_CHUNKED_TRANSFER;
    }
    else {
        this->bytesToRead = this->answerContentLength = size_t(this->httpParser->GetContentLength());
        if (this->answerContentLength == 0 && this->httpParser->GetHeaderField("Content-Length") == nullptr && !this->ConnectionIsKeelAlive()) {
            // If no content-Length is given and the connection is not defined as Keep-Alive, enable the mode to read until the
            // connection will be closed.
            this->readUntilConnectionClose = true;
            this->bytesToRead = UINT32_MAX; // Just to have something here...
        }
    }


    this->responseCode = this->httpParser->GetHTTPResultCode()->code;
    free(this->responseCodeDescription);
    this->responseCodeDescription = _strdup(this->httpParser->GetHTTPResultCode()->description);
    int httpCode = this->httpParser->GetHTTPResultCode()->code;
    switch (httpCode)
    {
    case 200:
    case 201:
    case 202:
    case 204:
        if (this->fakingAuth) {
            this->SetRequest();
            this->state = HTTP_CL_STATE_READY;
            this->DoSend(this->dataToSend, this->dataToSendSize);
        }
        else {
            this->HandleRequestAsOK(len, dataRead);
        }
        break;

    case 301:
    case 302:
    case 307:
    case 308:
        this->lastResult = HTTP_RESULT_REDIRECT;
        if (this->passErrorToUserFlags & HTTP_CL_ERR_REDIRECT) this->HandleRequestAsOK(len, dataRead, HTTP_RESULT_REDIRECT);
        else this->PrepareRedirect(len, dataRead, httpCode == 301 || httpCode == 308);
        break;

    case 400:
        if (this->passErrorToUserFlags & HTTP_CL_ERR_BAD_REQUEST) this->HandleRequestAsOK(len, dataRead, HTTP_RESULT_BAD_REQUEST);
        else {
            this->lastResult = HTTP_RESULT_BAD_REQUEST;
            this->shutdownReason = HTTP_BAD_REQUEST;
            this->Shutdown();
        }
        break;

    case 401:
        if (this->passErrorToUserFlags & HTTP_CL_ERR_AUTHENTICATION_FAILED) {
            this->HandleRequestAsOK(len, dataRead, http_result_t::HTTP_RESULT_AUTHENTICATION_ERROR);
        }
        else if (this->httpParser->GetContentLength() > 0 || this->httpParser->IsChunkedTransfer()) {
            //ASSERT(!httpParser->IsChunkedTransfer(), "HTTPClient::RecvAuthData() chunked 401 answer is not yet implemented :P");
            this->StartConsumeData(len, dataRead);
        }
        else {
            this->authToRead = 0;
            this->RecvAuthData(this->headerBuffer + dataRead, len - dataRead);
        }
        break;

    case 403:
        if (this->passErrorToUserFlags & HTTP_CL_ERR_FORBIDDEN) {
            this->HandleRequestAsOK(len, dataRead, HTTP_RESULT_NOT_FOUND);
        }
        else {
            this->shutdownReason = HTTP_UNHANDLED_HTTP_RESULT;
            this->Shutdown();
        }
        break;

    case 404:
        if (this->passErrorToUserFlags & HTTP_CL_ERR_NOT_FOUND)
            this->HandleRequestAsOK(len, dataRead, HTTP_RESULT_NOT_FOUND);
        else {
            this->lastResult = HTTP_RESULT_NOT_FOUND;
            this->shutdownReason = HTTP_NOT_FOUND;
            this->Shutdown();
        }
        break;
	    
    case 406:
        if (this->passErrorToUserFlags & HTTP_CL_ERR_NOT_ACCAPTABLE)
            this->HandleRequestAsOK(len, dataRead, HTTP_RESULT_NOT_FOUND);
        else {
	        this->shutdownReason = HTTP_UNHANDLED_HTTP_RESULT;
            this->Shutdown();
        }
	    break;    

    case 500:
        if (this->passErrorToUserFlags & HTTP_CL_ERR_INTERNAL_SERVER)
            this->HandleRequestAsOK(len, dataRead, HTTP_RESULT_INTERNAL_SERVER_ERROR);
        else {
            this->lastResult = HTTP_RESULT_INTERNAL_SERVER_ERROR;
            this->shutdownReason = HTTP_INTERNAL_SERVER_ERROR;
            this->Shutdown();
        }
        break;

    case 503:
        if (this->passErrorToUserFlags & HTTP_CL_ERR_SERVICE_UNAVAILABLE) {
            this->HandleRequestAsOK(len, dataRead, HTTP_RESULT_SERVICE_UNAVAILABLE);
        }
        else {
            this->lastResult = HTTP_RESULT_SERVICE_UNAVAILABLE;
            this->shutdownReason = HTTP_SERVICE_UNAVAILABLE;
            this->Shutdown();
        }
        break;

    default:
        if (this->passErrorToUserFlags & HTTP_CL_ERR_ALL) {
            this->HandleRequestAsOK(len, dataRead, HTTP_RESULT_UNKNOWN);
        }
        else {
            this->shutdownReason = HTTP_UNHANDLED_HTTP_RESULT;
            this->Shutdown();
        }
        break;
    }
}


void HTTPClient::DnsGetHostByNameResult(const char * addr, bool isIPv6)
{
    this->StopTimer();

    if (this->state == HTTP_CL_STATE_SHUTDOWN) {
        // No socket exists for now, so we can directly call the users shutdown callback...
        this->UserHTTPClientShutdown(this->shutdownReason);
        return;
    }
    this->state = HTTP_CL_STATE_RESOLVED_HOST_NAME;
    if (addr == 0) {
        if (this->proxy) {
            this->proxy = this->proxy->goNext();
            ConnectProxy();
            return;
        }
        this->shutdownReason = HTTP_ADDRESS_INVALID;
        this->Shutdown();
    }
    else {
        strcpy(this->proxy ? this->proxyIPAddress : this->hostIPAddress, addr);
        this->ConnectSocket();
    }
}


void HTTPClient::IoExec(void * execContext)
{
    // HTTPClient only uses SetExec to simulate an RecvResult() if there is
    // already a rest of data to read...
    DoRecv();
}


void HTTPClient::StartTimer()
{
    if (this->timeoutMs > 0 && this->state != HTTP_CL_STATE_SHUTDOWN && !this->timer.IsRunning()) this->timer.Start(this->timeoutMs);
}


void HTTPClient::StopTimer()
{
    if (this->timeoutMs != 0) this->timer.Cancel();
}


void HTTPClient::TimerOnTimeout(ITimer * timer)
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::TimerOnTimeout shutdown connection", this, this->user);
    if (this->state == HTTP_CL_STATE_RESOLVED_HOST_NAME) this->timeoutOnConnect = true;
    this->shutdownReason = http_shutdown_reason_t::HTTP_SHUTDOWN_TIMEOUT;
    this->Shutdown();
}


void HTTPClient::UserHTTPClientConnectComplete()
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::HTTPClientConnectComplete", this, this->user);
    this->user->HTTPClientConnectComplete(this);
}


void HTTPClient::UserHTTPClientShutdown(http_shutdown_reason_t reason)
{
    // The header fields must be deleted in case the app reuses the http client instance (e. G. when implementing an own OAuth2 authentication).
    ReleaseCustomHeaderFields(this->customFieldList);

    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::HTTPClientShutdown reason:%s", this, this->user, HTTPShutdownReasonToStr(reason));
    this->state = HTTP_CL_STATE_OFFLINE;
    this->user->HTTPClientShutdown(this, reason);
}


void HTTPClient::UserHTTPClientSendResult()
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::HTTPClientSendResult", this, this->user);
    this->sendCnt--;
    this->user->HTTPClientSendResult(this);
}


void HTTPClient::UserHTTPClientRecvResult(bool transferComplete)
{
    byte * tmpBuf = this->userBuffer;
    size_t tmpLen = this->userBufferAlreadyRead;
    this->userBuffer = nullptr;
    this->userBufferSize = 0;
    this->userBufferAlreadyRead = 0;

    if (this->consumingData) {
        if (transferComplete) {
            free(tmpBuf);
            this->consumingData = false;
            this->userBuffer = this->savedUserBuffer;
            this->userBufferSize = this->savedUserBufferSize;
            this->state = this->savedState;
            this->lastResult = this->savedLastResult;
            if (this->httpRedirectData != nullptr) {
                this->FollowRedirect();
            }
            else {
                this->Authenticate();
            }
        }
        else {
            this->userBuffer = tmpBuf;
            this->userBufferSize = 1024;
            this->DoRecv();
        }
    }
    else {
        this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::HTTPClientRecvResult len:%i transferComplete:%i", this, this->user, tmpLen, transferComplete);
        this->log->HexDump(LOG_HTTP_CLIENT, tmpBuf, tmpLen);
        if (transferComplete) {
            dword sendCntTmp = this->sendCnt;
            // work with a copy from sendCnt, as sendCnt might be incremented again within UserHTTPClientSendResult
            while (sendCntTmp != 0) { // trigger a send result for each previous Send call
                this->UserHTTPClientSendResult();
                --sendCntTmp;
            }

            ReleaseCustomHeaderFields(this->customFieldList);

            if (this->state != HTTP_CL_STATE_SHUTDOWN) {
                this->state = HTTP_CL_STATE_READY;
            }
        }
        if (this->shutdownReason == HTTP_INTERNAL_SERVER_ERROR) {
            this->Shutdown();
        }
        else {
            this->user->HTTPClientRecvResult(this, tmpBuf, tmpLen, transferComplete);
        }
    }
}


void HTTPClient::UserHTTPClientRecvCanceled(byte * buf)
{
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::UserHTTPClientRecvCanceled", this, this->user);
    this->user->HTTPClientRecvCanceled(this, buf);
}


void HTTPClient::UserHTTPClientRequestComplete()
{
    if (this->sendCnt != 0) {
        this->UserHTTPClientSendResult();
    }
    this->log->Log(LOG_HTTP_CLIENT, "HTTPClient(%08x,%08x)::HTTPClientRequestComplete", this, this->user);
    bool doRecv = (this->userBuffer != nullptr);
    this->user->HTTPClientRequestComplete(this);
    
    if (doRecv) {   // app already issued a read, which we now process
        this->DoRecv(); 
    }
}


void HTTPClient::SetTimeout(dword timeoutMs)
{
    if (this->timeoutMs > 0 && timeoutMs == 0) this->timer.Cancel();
    this->timeoutMs = timeoutMs;
}

void HTTPClient::DebugPrintHTTPHeader(const char * caption, bool showReceivedHeader)
{
    if (this->log->LogFlagSet(LOG_HTTP_CLIENT)) {
        debug->printf("HTTPClient(%08x,%08x)::%s", this, this->user, caption);

        if (showReceivedHeader) {
            const HTTPLib::http_result_t * res = this->httpParser->GetHTTPResultCode();
            debug->printf("    HTTP/%u.%u %u %s", this->httpParser->GetMajorVersion(), this->httpParser->GetMinorVersion(), res == nullptr ? 0 : res->code, res == nullptr ? "" : res->description);
            byte * data = nullptr, * dataTmp = nullptr;
            size_t dataSize = 0;
            this->httpParser->GetFieldListCopy(data, dataSize);
            dataTmp = data;

            std::string field;
            std::string value;
            while (dataSize > 0) {
                field = reinterpret_cast<char *>(data);
                data += field.size() + 1;
                dataSize -= field.size() + 1;
                value = reinterpret_cast<char *>(data);
                data += value.size() + 1;
                dataSize -= value.size() + 1;
                debug->printf("    %s: %s", field.c_str(), value.c_str());
            }
            free(dataTmp);
        }
        else {
            const char * buf = this->httpHeader->GetHeaderData();
            const char * end = strchr(buf, '\r');
            while (end != nullptr && buf != end) {
                debug->printf("    %.*s", end - buf, buf);
                buf = end + 2;
                end = strchr(buf, '\r');
            }
        }
    }
}
