juce_linux_Network.cpp

#1

i’ve coded one platform specific file, the one that makes the URL class working. actually i’ve introduced a http URL GET based on sockets,
with redirects and proxy support. actually the only thing that isn’t supported is the chunking Transfer-Encoding, but this will soon follows.
hey jules, give it an eye, and feel free to take something for the new juce release !

UPDATED:

  • now supports http proxies with no authentication
/*
  ==============================================================================

   This file is part of the JUCE library - "Jules' Utility Class Extensions"
   Copyright 2004-6 by Raw Material Software ltd.

  ------------------------------------------------------------------------------

   JUCE can be redistributed and/or modified under the terms of the
   GNU General Public License, as published by the Free Software Foundation;
   either version 2 of the License, or (at your option) any later version.

   JUCE is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with JUCE; if not, visit www.gnu.org/licenses or write to the
   Free Software Foundation, Inc., 59 Temple Place, Suite 330,
   Boston, MA 02111-1307 USA

  ------------------------------------------------------------------------------

   If you'd like to release a closed-source product which uses JUCE, commercial
   licenses are also available: visit www.rawmaterialsoftware.com/juce for
   more information.

  ==============================================================================
*/

#include "linuxincludes.h"

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <ctype.h>
#include <errno.h>
#include <assert.h>
#include <signal.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/wait.h>

#include "../../../src/juce_core/basics/juce_StandardHeader.h"

BEGIN_JUCE_NAMESPACE

#include "../../../src/juce_core/text/juce_String.h"
#include "../../../src/juce_core/basics/juce_SystemStats.h"

#define HEADER_BUF_SIZE     4096
#define MAX_REDIRECTS       3
#define HTTP_PROTO          T("http://")
#define CRLF                T("\r\n")


//==============================================================================
class JUCE_UrlStream
{
public:

    //==============================================================================
    JUCE_UrlStream ()
        : sock (-1),
          timeout (10),
          redirectCount (0),
          chunked (0),
          responseCode (0),
          contentLength (0)
    {}

    ~JUCE_UrlStream ()
    {
        if (sock >= 0)
            close (sock);
        sock = -1;
    }

    //==============================================================================
    bool open (const String& url,
               const String& optionalPostText,
               const bool isPost)
    {
        if (sock >= 0)
            close (sock);
        sock = -1;

        // parse the url
        bool isProxied = false;

        String hostName, hostPath,
               proxyURL, proxyName, proxyPath;
        int hostPort, proxyPort;

        if (! parseURL (url, hostName, hostPath, hostPort))
        {
            return false;
        }

        // check for http_proxy
        char* proxyEnv = getenv ("http_proxy");
        if (proxyEnv != 0)
        {
            proxyURL = proxyEnv;
            isProxied = true;

            if (! parseURL (proxyURL, proxyName, proxyPath, proxyPort))
            {
                return false;
            }
        }

        // initialize host name
        struct hostent* hp;
        struct sockaddr_in sa;

        hp = gethostbyname (isProxied ? ((const char*) proxyName) : ((const char*) hostName));
        if (hp == NULL) return false;

        // initialize the socket
        zerostruct (sa);
        memcpy ((char *)&sa.sin_addr, (char *)hp->h_addr, hp->h_length);
        sa.sin_family = hp->h_addrtype;
        sa.sin_port = htons (isProxied ? proxyPort : hostPort);

        // open the socket
        sock = socket (hp->h_addrtype, SOCK_STREAM, 0);
        if (sock == -1) return false;

        // tune send options
        int windowSize = 32768;   // we could tune this, but for now...
        setsockopt (sock, SOL_SOCKET, SO_RCVBUF, (char*) &windowSize, sizeof(windowSize));
        setsockopt (sock, SOL_SOCKET, SO_KEEPALIVE, 0, 0);

        // connect the socket
        if (connect (sock, (struct sockaddr *)&sa, sizeof(sa)) == -1)
            return false;

        // do real request
        String header;
        if (doRequest (hostName, hostPath, hostPort, optionalPostText, isPost, isProxied)
            && (header = doResponse ()) != String::empty)
        {
//            DBG (header);

            responseCode = JUCE_UrlStream::parseResponseCode (header);
            contentLength = JUCE_UrlStream::parseContentLength (header);
            chunked = JUCE_UrlStream::parseTransferEncoding (header).equalsIgnoreCase ("chunked");

            String location = JUCE_UrlStream::parseLocation (header);

//            DBG (String (responseCode));
//            DBG (String (contentLength));
//            DBG (String (chunked));

            if (responseCode >= 300
                && responseCode < 400
                && location != String::empty)
            {
                if (! location.startsWithIgnoreCase (HTTP_PROTO))
                    location = HTTP_PROTO + location;

//                DBG ("[" + location + "]");

                if (redirectCount++ < MAX_REDIRECTS)
                    return open (location, optionalPostText, isPost);
            }
            else
            {
                redirectCount = 0;
                return true;
            }
        }

        return false;
    }

    //==============================================================================
    int read (void* buffer, int bytesToRead)
    {
        fd_set rfds;
        struct timeval tv;

        FD_ZERO (&rfds);
        FD_SET (sock, &rfds);

        tv.tv_sec = timeout;
        tv.tv_usec = 0;

        if (select (sock + 1, &rfds, NULL, NULL, &tv) <= 0)
        {
            DBG ("read timeout");
            return 0; // timed out
        }

        int bytesRead = recv (sock, buffer, bytesToRead, MSG_WAITALL);

        return bytesRead;
    }

    //==============================================================================
    int getStatusCode ()
    {
        return responseCode;
    }

protected:

    //==============================================================================
    bool doRequest (const String& hostName,
                    const String& hostPath,
                    const int hostPort,
                    const String& optionalPostText,
                    const bool isPost,
                    const bool isProxied)
    {
        String header;
        if (isPost)  header << "POST ";
        else         header << "GET ";

        if (isProxied)
            header << "http://" << hostName << ":" << hostPort << hostPath << " HTTP/1.1" << CRLF;
        else
            header << hostPath << " HTTP/1.1" << CRLF;

        header << "Host: " << hostName << ":" << hostPort << CRLF;
        header << "User-Agent: JUCE/"
                << JUCE_MAJOR_VERSION << "."
                << JUCE_MINOR_VERSION << " (Linux)" << CRLF;

        if (isPost)
        {
            header  << "Content-type: application/x-www-form-urlencoded" << CRLF
                    << "Content-length: " << optionalPostText.length() << CRLF;
            if (optionalPostText.length())
                header << optionalPostText << CRLF;
        }
        header << "Connection: Close" << CRLF << CRLF;

        // DBG (header);

        return send (sock, (const char*) header, header.length(), 0) == header.length();
    }

    //==============================================================================
    String doResponse ()
    {
        fd_set rfds;
        struct timeval tv;
        int bytesRead = 0, newlines = 0;
        char headerBuffer [HEADER_BUF_SIZE];
        char* headerPtr = &headerBuffer[0];

        while (newlines != 2 && bytesRead < HEADER_BUF_SIZE)
        {
            FD_ZERO (&rfds);
            FD_SET (sock, &rfds);
            tv.tv_sec = timeout;
            tv.tv_usec = 0;
            if (select (sock + 1, &rfds, NULL, NULL, &tv) <= 0)
                return String::empty; // timed out

            if (recv (sock, headerPtr, 1, 0) == -1)
                return String::empty; // receive error

            bytesRead++;

            if (*headerPtr == '\r') {
                headerPtr++;
                continue;
            }
            else if (*headerPtr == '\n') // LF is the separator
                newlines++;
            else
                newlines = 0;

            headerPtr++;
        }

        headerPtr -= 3; // Snip the trailing LF's
        *headerPtr = '\0';

        String header (&headerBuffer[0]);
        if (header.startsWithIgnoreCase (T("HTTP/")))
            return header;

        return String::empty; // not http header
    }

    //==============================================================================
    static bool parseURL (const String& url, String& host, String& path, int& port)
    {
        if (url == String::empty
            || ! url.startsWithIgnoreCase (HTTP_PROTO))
            return false;

        int pos = String (HTTP_PROTO).length();

        int nextColon = url.indexOfChar (pos, ':');
        int nextSlash = url.indexOfChar (pos, '/');
//        int nextMark = url.indexOfChar (pos, '?');

        // optional port specification
        if (nextColon >= 0
            && ((nextSlash >= 0 && nextColon < nextSlash) || nextSlash < 0))
        {
            if (nextSlash >= 0)
                port = url.substring (nextColon + 1, nextSlash).getIntValue ();
            else
                port = url.substring (nextColon + 1).getIntValue ();
        }
        else
            port = 80;

        // only host in URL, assume '/' for path
        if (nextSlash < 0)
        {
            if (nextColon >= 0)
                host = url.substring (pos, nextColon);
//            else if (nextMark >= 0)
//                host = url.substring (pos, nextMark);
            else
                host = url.substring (pos);
            path = T("/");

 //           DBG ("[" + host + "]");
 //           DBG ("[" + String(port) + "]");
 //           DBG ("[" + path + "]");

            return true;
        }

        // normal parsing: both host and path exist
        if (nextColon >= 0)
            host = url.substring (pos, nextColon);
//        else if (nextMark >= 0)
//            host = url.substring (pos, nextMark);
        else
            host = url.substring (pos, nextSlash);

        path = url.substring (nextSlash);

//        DBG ("(" + host + ")");
//        DBG ("(" + String(port) + ")");
//        DBG ("(" + path + ")");

        return true;
    }

    //==============================================================================
    static int parseResponseCode (const String& header)
    {
        int pos = header.indexOfChar (' ') + 1;
        if (pos >= 0)
            return header.substring (pos, pos + 3).trim().getIntValue ();
        return 0;
    }

    //==============================================================================
    static String parseLocation (const String& header)
    {
        int pos = header.indexOfIgnoreCase (T("Location:"));
        if (pos >= 0)
        {
            int nextNewLine = header.indexOf (pos + 9, CRLF);
            if (nextNewLine >= 0)
                return header.substring (pos + 9, nextNewLine).trim();
        }
        return String::empty;
    }

    //==============================================================================
    static String parseTransferEncoding (const String& header)
    {
        int pos = header.indexOfIgnoreCase (T("Transfer-Encoding:"));
        if (pos >= 0)
        {
            int nextNewLine = header.indexOf (pos + 18, CRLF);
            if (nextNewLine >= 0)
                return header.substring (pos + 18, nextNewLine).trim();
        }
        return String::empty;
    }

    //==============================================================================
    static String parseContentType (const String& header, String& encoding)
    {
        int pos = header.indexOfIgnoreCase (T("Content-Type:"));
        if (pos >= 0)
        {
            int nextColon = header.indexOfChar (pos + 13, ':');
            if (nextColon >= 0)
            {
                // @TODO - check for charset
            }

            int nextNewLine = header.indexOf (pos + 13, CRLF);
            if (nextNewLine >= 0)
                return header.substring (pos + 13, nextNewLine).trim();
        }
        return String::empty;
    }

    //==============================================================================
    static int parseContentLength (const String& header)
    {
        int pos = header.indexOfIgnoreCase (T("Content-Length:"));
        if (pos >= 0)
        {
            int nextNewLine = header.indexOf (pos + 15, CRLF);
            if (nextNewLine >= 0)
                return header.substring (pos + 15, nextNewLine).trim().getIntValue();
        }
        return 0;
    }

private:

    int sock;
    int timeout;
    int redirectCount;
    int chunked;
    int responseCode;
    int contentLength;
};




//==============================================================================
bool juce_isOnLine()
{
    return true;
}

void* juce_openInternetFile (const String& url,
                             const String& optionalPostText,
                             const bool isPost)
{
    JUCE_UrlStream* stream = new JUCE_UrlStream ();

    if (stream->open (url, optionalPostText, isPost))
        return stream;
    else
        delete stream;

    return 0;
}

void juce_closeInternetFile (void* handle)
{
    JUCE_UrlStream* stream = (JUCE_UrlStream*) handle;
    if (stream != 0)
    {
        delete stream;
    }
}

int juce_getStatusCodeFor (void* fileHandle)
{
    JUCE_UrlStream* stream = (JUCE_UrlStream*) fileHandle;
    if (stream != 0)
    {
        return stream->getStatusCode ();
    }
    return 0;
}

int juce_readFromInternetFile (void* handle, void* buffer, int bytesToRead)
{
    JUCE_UrlStream* stream = (JUCE_UrlStream*) handle;
    if (stream != 0)
    {
        return stream->read (buffer, bytesToRead);
    }
    return 0;
}

int juce_seekInInternetFile (void* handle, int newPosition)
{
    return 0;
}

int SystemStats::getMACAddresses (int64* addresses, int maxNum)
{
    return 0;
}

END_JUCE_NAMESPACE
#2

Hey, thanks for that, kraken. I’d actually forgotten all about needing to implement that stuff on linux! I’ll take a look as soon as I get a chance.

#3

yes i needed that URL worked in linux also: actually is not complete (lots of sites have chunked encoding and that possibility is not handled in my code, also there is no check for environment variable http_proxy but i’m writing them out as speaking) but for basic sites is working good enough.
i think i should have used juce::Socket instead of the standard sys/socket.h plain C library. but again this shouldn’t matter too much…

#4

What is wrong with using urllib as the backend?
Or just opening the address as a file, which, in any non-windows system I use, will cause it to fetch the webpage/file…

#5

mmh, surely i and jules don’t want to add urllib or any other libraries that are not in anyone linux distro by default. at least not in mine…
just a call to fopen would do it ? it handles redirects ? it handle chunked encoding ? it posts data instead of get also ?
can you show me the right way to do it with file ?

gr8 to know that something can be done in a simpler way. i’ll try it.

#6

I’m used to just doing pure socket read/writes, so open a socket (which is handled as file in most things non-windows), write the full http header to retrieve what you want (so it can be get, post, whatever), and read back. Gets a little more complicated if you want it async, but still simple enough.

#7

actually, i’m just doing this…

#8

edit!

#9