source: trunk/third/gtkhtml/src/htmltokenizer.c @ 18136

/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
/*
    Copyright (C) 1997 Martin Jones (mjones@kde.org)
              (C) 1997 Torben Weis (weis@kde.org)
              (C) 1999 Anders Carlsson (andersca@gnu.org)
              (C) 2000 Helix Code, Inc., Radek Doulik (rodo@helixcode.com)
              (C) 2001 Ximian, Inc.

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Library General Public License for more details.

    You should have received a copy of the GNU Library General Public License
    along with this library; see the file COPYING.LIB.  If not, write to
    the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
    Boston, MA 02111-1307, USA.
*/

/* The HTML Tokenizer */
#include <config.h>
#include <ctype.h>
#include <gnome.h>
#include <gal/unicode/gunicode.h>
#include "htmltokenizer.h"
#include "htmlentity.h"

enum {
        HTML_TOKENIZER_BEGIN_SIGNAL,
        HTML_TOKENIZER_END_SIGNAL,
        HTML_TOKENIZER_LAST_SIGNAL
};

static guint html_tokenizer_signals[HTML_TOKENIZER_LAST_SIGNAL] = { 0 };

#define TOKEN_BUFFER_SIZE (1 << 10)

typedef struct _HTMLBlockingToken HTMLBlockingToken;
typedef struct _HTMLTokenBuffer   HTMLTokenBuffer;
typedef enum { Table }            HTMLTokenType;

struct _HTMLTokenBuffer {
        gint size;
        gint used;
        gchar * data;
};
struct _HTMLTokenizerPrivate {

        /* token buffers list */
        GList *token_buffers;

        /* current read_buf position in list */
        GList *read_cur;

        /* current read buffer */
        HTMLTokenBuffer * read_buf;
        HTMLTokenBuffer * write_buf;

        /* position in the read_buf */
        gint read_pos;

        /* non-blocking and blocking unreaded tokens in tokenizer */
        gint tokens_num;
        gint blocking_tokens_num;

        gchar *dest;
        gchar *buffer;
        gint size;

        gboolean skipLF; /* Skip the LF par of a CRLF sequence */

        gboolean tag; /* Are we in an html tag? */
        gboolean tquote; /* Are we in quotes in an html tag? */
        gboolean startTag;
        gboolean comment; /* Are we in a comment block? */
        gboolean title; /* Are we in a <title> block? */
        gboolean style; /* Are we in a <style> block? */
        gboolean script; /* Are we in a <script> block? */
        gboolean textarea; /* Are we in a <textarea> block? */
        gint     pre; /* Are we in a <pre> block? */
        gboolean select; /* Are we in a <select> block? */
        gboolean charEntity; /* Are we in an &... sequence? */
        gboolean extension; /* Are we in an <!-- +GtkHTML: sequence? */
 
        enum {
                NoneDiscard = 0,
                SpaceDiscard,
                LFDiscard
        } discard;

        enum {
                NonePending = 0,
                SpacePending,
                LFPending,
                TabPending
        } pending;


        gchar searchBuffer[20];
        gint searchCount;
        gint searchGtkHTMLCount;
        gint searchExtensionEndCount;

        gchar *scriptCode;
        gint scriptCodeSize;
        gint scriptCodeMaxSize;

        GList *blocking; /* Blocking tokens */

        const gchar *searchFor;
        gboolean utf8;
        gchar utf8_buffer[7];
        gint utf8_length;
};

static const gchar *commentStart = "<!--";
static const gchar *scriptEnd = "</script>";
static const gchar *styleEnd = "</style>";
static const gchar *gtkhtmlStart = "+gtkhtml:";

enum quoteEnum {
        NO_QUOTE = 0,
        SINGLE_QUOTE,
        DOUBLE_QUOTE
};

/* private tokenizer functions */
static void           html_tokenizer_reset        (HTMLTokenizer *t);
static void           html_tokenizer_add_pending  (HTMLTokenizer *t);
static void           html_tokenizer_append_token (HTMLTokenizer *t,
                                                   const gchar *string,
                                                   gint len);
static void           html_tokenizer_append_token_buffer (HTMLTokenizer *t,
                                                          gint min_size);

/* default implementations of tokenization functions */
static void     html_tokenizer_destructor          (GtkObject *);
static void     html_tokenizer_real_begin           (HTMLTokenizer *, gchar *content_type);
static void     html_tokenizer_real_write           (HTMLTokenizer *, const gchar *str, size_t size);
static void     html_tokenizer_real_end             (HTMLTokenizer *);
static gchar   *html_tokenizer_real_peek_token      (HTMLTokenizer *);
static gchar   *html_tokenizer_real_next_token      (HTMLTokenizer *);
static gboolean html_tokenizer_real_has_more_tokens (HTMLTokenizer *);

static HTMLTokenizer *html_tokenizer_real_clone     (HTMLTokenizer *);

/* blocking tokens */
static gchar             *html_tokenizer_blocking_get_name   (HTMLTokenizer  *t);
static void               html_tokenizer_blocking_pop        (HTMLTokenizer  *t);
static void               html_tokenizer_blocking_push       (HTMLTokenizer  *t,
                                                              HTMLTokenType   tt);
static void               html_tokenizer_tokenize_one_char   (HTMLTokenizer  *t,
                                                              const gchar  **src);

static void               add_unichar(HTMLTokenizer *t, gunichar wc);

static GtkObjectClass *parent_class = NULL;

static void
html_tokenizer_class_init (HTMLTokenizerClass *klass)
{
        GtkObjectClass *object_class = (GtkObjectClass *) klass;

        parent_class = gtk_type_class (GTK_TYPE_OBJECT);

        html_tokenizer_signals[HTML_TOKENIZER_BEGIN_SIGNAL] =
                gtk_signal_new ("begin",
                                GTK_RUN_LAST,
                                object_class->type,
                                GTK_SIGNAL_OFFSET (HTMLTokenizerClass, begin),
                                gtk_marshal_NONE__POINTER,
                                GTK_TYPE_NONE,
                                1, GTK_TYPE_POINTER);

        html_tokenizer_signals[HTML_TOKENIZER_END_SIGNAL] =
                gtk_signal_new ("end",
                                GTK_RUN_LAST,
                                object_class->type,
                                GTK_SIGNAL_OFFSET (HTMLTokenizerClass, end),
                                gtk_marshal_NONE__NONE,
                                GTK_TYPE_NONE,
                                0);

        gtk_object_class_add_signals (object_class, html_tokenizer_signals, HTML_TOKENIZER_LAST_SIGNAL);

        object_class->destroy = html_tokenizer_destructor;

        klass->begin      = html_tokenizer_real_begin;
        klass->end        = html_tokenizer_real_end;

        klass->write      = html_tokenizer_real_write;
        klass->peek_token = html_tokenizer_real_peek_token;
        klass->next_token = html_tokenizer_real_next_token;
        klass->has_more   = html_tokenizer_real_has_more_tokens;
        klass->clone      = html_tokenizer_real_clone;
}

static void
html_tokenizer_init (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p;

        t->priv = p = g_new0 (struct _HTMLTokenizerPrivate, 1);

        p->token_buffers = NULL;
        p->read_cur  = NULL;
        p->read_buf  = NULL;
        p->write_buf = NULL;
        p->read_pos  = 0;

        p->dest = NULL;
        p->buffer = NULL;
        p->size = 0;

        p->skipLF = FALSE;
        p->tag = FALSE;
        p->tquote = FALSE;
        p->startTag = FALSE;
        p->comment = FALSE;
        p->title = FALSE;
        p->style = FALSE;
        p->script = FALSE;
        p->textarea = FALSE;
        p->pre = 0;
        p->select = FALSE;
        p->charEntity = FALSE;
        p->extension = FALSE;

        p->discard = NoneDiscard;
        p->pending = NonePending;

        memset (p->searchBuffer, 0, sizeof (p->searchBuffer));
        p->searchCount = 0;
        p->searchGtkHTMLCount = 0;

        p->scriptCode = NULL;
        p->scriptCodeSize = 0;
        p->scriptCodeMaxSize = 0;

        p->blocking = NULL;

        p->searchFor = NULL;
}

static void
html_tokenizer_destructor (GtkObject *obj)
{
        HTMLTokenizer *t = HTML_TOKENIZER (obj);

        html_tokenizer_reset (t);

        g_free (t->priv);
        t->priv = NULL;
}

GtkType
html_tokenizer_get_type (void)
{
        static GtkType html_tokenizer_type = 0;

        if (!html_tokenizer_type) {
                static const GtkTypeInfo html_tokenizer_info = {
                        "HTMLTokenizer",
                        sizeof (HTMLTokenizer),
                        sizeof (HTMLTokenizerClass),
                        (GtkClassInitFunc) html_tokenizer_class_init,
                        (GtkObjectInitFunc) html_tokenizer_init,
                        NULL, NULL,
                        (GtkClassInitFunc) NULL
                };
                html_tokenizer_type = gtk_type_unique (GTK_TYPE_OBJECT, &html_tokenizer_info);
        }

        return html_tokenizer_type;
}

static HTMLTokenBuffer *
html_token_buffer_new (gint size)
{
        HTMLTokenBuffer *nb = g_new (HTMLTokenBuffer, 1);

        nb->data = g_new (gchar, size);
        nb->size = size;
        nb->used = 0;

        return nb;
}

static void
html_token_buffer_destroy (HTMLTokenBuffer *tb)
{
        g_free (tb->data);
        g_free (tb);
}

static gboolean
html_token_buffer_append_token (HTMLTokenBuffer * buf, const gchar *token, gint len)
{

        /* check if we have enough free space */
        if (len + 1 > buf->size - buf->used) {
                return FALSE;
        }

        /* copy token and terminate with zero */
        strncpy (buf->data + buf->used, token, len);
        buf->used += len;
        buf->data [buf->used] = 0;
        buf->used ++;

        return TRUE;
}

HTMLTokenizer *
html_tokenizer_new (void)
{
        return (HTMLTokenizer *) gtk_type_new (HTML_TYPE_TOKENIZER);
}

void
html_tokenizer_destroy (HTMLTokenizer *t)
{
        g_return_if_fail (t && HTML_IS_TOKENIZER (t));
        
        gtk_object_unref (GTK_OBJECT (t));
}

static gchar *
html_tokenizer_real_peek_token (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        gchar *token;

        g_assert (p->read_buf);

        if (p->read_buf->used > p->read_pos) {
                token = p->read_buf->data + p->read_pos;
        } else {
                GList *next;
                HTMLTokenBuffer *buffer;

                g_assert (p->read_cur);
                g_assert (p->read_buf);

                /* lookup for next buffer */
                next = p->read_cur->next;
                g_assert (next);

                buffer = (HTMLTokenBuffer *) next->data;

                g_return_val_if_fail (buffer->used != 0, NULL);

                /* finally get first token */
                token = buffer->data;
        }
        
        return token;
}
        
static gchar *
html_tokenizer_real_next_token (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        gchar *token;

        g_assert (p->read_buf);

        /* token is in current read_buf */
        if (p->read_buf->used > p->read_pos) {
                token = p->read_buf->data + p->read_pos;
                p->read_pos += strlen (token) + 1;
        } else {
                GList *new;

                g_assert (p->read_cur);
                g_assert (p->read_buf);

                /* lookup for next buffer */
                new = p->read_cur->next;
                g_assert (new);

                /* destroy current buffer */
                p->token_buffers = g_list_remove (p->token_buffers, p->read_buf);
                html_token_buffer_destroy (p->read_buf);

                p->read_cur = new;
                p->read_buf = (HTMLTokenBuffer *) new->data;

                g_return_val_if_fail (p->read_buf->used != 0, NULL);

                /* finally get first token */
                token = p->read_buf->data;
                p->read_pos = strlen (token) + 1;
        }

        p->tokens_num--;
        g_assert (p->tokens_num >= 0);

        return token;
}

static gboolean
html_tokenizer_real_has_more_tokens (HTMLTokenizer *t)
{
        return t->priv->tokens_num > 0;
}

static HTMLTokenizer *
html_tokenizer_real_clone (HTMLTokenizer *t)
{
        return html_tokenizer_new ();
}

static void
html_tokenizer_reset (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        GList *cur = p->token_buffers;

        /* free remaining token buffers */
        while (cur) {
                g_assert (cur->data);
                html_token_buffer_destroy ((HTMLTokenBuffer *) cur->data);
                cur = cur->next;
        }

        /* reset buffer list */
        g_list_free (p->token_buffers);
        p->token_buffers = p->read_cur = NULL;
        p->read_buf = p->write_buf = NULL;
        p->read_pos = 0;

        /* reset token counters */
        p->tokens_num = p->blocking_tokens_num = 0;

        if (p->buffer)
                g_free (p->buffer);
        p->buffer = NULL;
        p->dest = NULL;
        p->size = 0;

        if (p->scriptCode)
                g_free (p->scriptCode);
        p->scriptCode = NULL;
}

static gint
charset_is_utf8 (gchar *content_type)
{
        return content_type && strstr (content_type, "charset=utf-8") != NULL;
}

static void
html_tokenizer_real_begin (HTMLTokenizer *t, gchar *content_type)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        html_tokenizer_reset (t);

        p->dest = p->buffer;
        p->tag = FALSE;
        p->pending = NonePending;
        p->discard = NoneDiscard;
        p->pre = 0;
        p->script = FALSE;
        p->style = FALSE;
        p->skipLF = FALSE;
        p->select = FALSE;
        p->comment = FALSE;
        p->textarea = FALSE;
        p->startTag = FALSE;
        p->extension = FALSE;
        p->tquote = NO_QUOTE;
        p->searchCount = 0;
        p->searchGtkHTMLCount = 0;
        p->title = FALSE;
        p->charEntity = FALSE;
        
        p->utf8 = charset_is_utf8 (content_type);
        p->utf8_length = 0;
#if 0
        if (p->utf8) 
                g_warning ("Trying UTF-8");
        else 
                g_warning ("Trying ISO-8859-1");
#endif

}

static void
destroy_blocking (gpointer data, gpointer user_data)
{
        g_free (data);
}

static void
html_tokenizer_real_end (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p = t->priv;

        if (p->buffer == 0)
                return;

        if (p->dest > p->buffer) {
                html_tokenizer_append_token (t, p->buffer, p->dest - p->buffer);
        }

        g_free (p->buffer);     

        p->buffer = NULL;
        p->dest = NULL;
        p->size = 0;

        if (p->blocking) {
                g_list_foreach (p->blocking, destroy_blocking, NULL);
                p->tokens_num += p->blocking_tokens_num;
                p->blocking_tokens_num = 0;
        }
        p->blocking = NULL;
}

static void
html_tokenizer_append_token (HTMLTokenizer *t, const gchar *string, gint len)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        if (len < 1)
                return;

        /* allocate first buffer */
        if (p->write_buf == NULL)
                html_tokenizer_append_token_buffer (t, len);

        /* try append token to current buffer, if not successful, create append new token buffer */
        if (!html_token_buffer_append_token (p->write_buf, string, len)) {
                html_tokenizer_append_token_buffer (t, len+1);
                /* now it must pass as we have enough space */
                g_assert (html_token_buffer_append_token (p->write_buf, string, len));
        }

        if (p->blocking) {
                p->blocking_tokens_num++;
        } else {
                p->tokens_num++;
        }
}

static void
html_tokenizer_append_token_buffer (HTMLTokenizer *t, gint min_size)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        HTMLTokenBuffer *nb;
        gint size = TOKEN_BUFFER_SIZE;

        if (min_size > size)
                size = min_size + (min_size >> 2);

        /* create new buffer and add it to list */
        nb = html_token_buffer_new (size);
        p->token_buffers = g_list_append (p->token_buffers, nb);

        /* this one is now write_buf */
        p->write_buf = nb;

        /* if we don't have read_buf already set it to this one */
        if (p->read_buf == NULL) {
                p->read_buf = nb;
                p->read_cur = p->token_buffers;
        }
}

/* EP CHECK: OK.  */
static void
html_tokenizer_add_pending (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        if (p->tag || p->select) {
                add_unichar (t, ' ');
        }
        else if (p->textarea) {
                if (p->pending == LFPending) 
                        add_unichar (t, '\n');
                else
                        add_unichar (t, ' ');
        }
        else if (p->pre) {
                switch (p->pending) {
                case SpacePending:
                        add_unichar (t, ' ');
                        break;
                case LFPending:
                        if (p->dest > p->buffer) {
                                html_tokenizer_append_token (t, p->buffer, p->dest - p->buffer);
                        }
                        p->dest = p->buffer;
                        add_unichar (t, TAG_ESCAPE);
                        add_unichar (t, '\n');
                        html_tokenizer_append_token (t, p->buffer, 2);
                        p->dest = p->buffer;
                        break;
                case TabPending:
                        add_unichar (t, '\t');
                        break;
                default:
                        g_warning ("Unknown pending type: %d\n", (gint) p->pending);
                        break;
                }
        }
        else {
                add_unichar (t, ' ');
        }
        
        p->pending = NonePending;
}

static void
prepare_enough_space (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        if ((p->dest - p->buffer + 32) > p->size) {
                guint off = p->dest - p->buffer;

                p->size  += (p->size >> 2) + 32;
                p->buffer = g_realloc (p->buffer, p->size);
                p->dest   = p->buffer + off;
        }
}

static void
in_comment (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;

        if (**src == '-') {                  /* Look for "-->" */
                if (p->searchCount < 2)
                        p->searchCount++;
        } else if (p->searchCount == 2 && (**src == '>')) {
                p->comment = FALSE;          /* We've got a "-->" sequence */
        } else if (tolower (**src) == gtkhtmlStart [p->searchGtkHTMLCount]) {
                if (p->searchGtkHTMLCount == 8) {
                        p->extension    = TRUE;
                        p->comment = FALSE;
                        p->searchCount = 0;
                        p->searchExtensionEndCount = 0;
                        p->searchGtkHTMLCount = 0;
                } else
                        p->searchGtkHTMLCount ++;
        } else {
                p->searchGtkHTMLCount = 0;
                if (p->searchCount < 2)
                        p->searchCount = 0;
        }

        (*src)++;
}

static inline void
extension_one_char (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        p->extension = FALSE;
        html_tokenizer_tokenize_one_char (t, src);
        p->extension = TRUE;
}

static void
in_extension (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;

        /* check for "-->" */
        if (!p->tquote && **src == '-') {
                if (p->searchExtensionEndCount < 2)
                        p->searchExtensionEndCount ++;
                (*src) ++;
        } else if (!p->tquote && p->searchExtensionEndCount == 2 && **src == '>') {
                p->extension = FALSE;
                (*src) ++;
        } else {
                if (p->searchExtensionEndCount > 0) {
                        if (p->extension) {
                                const gchar *c = "-->";

                                while (p->searchExtensionEndCount) {
                                        extension_one_char (t, &c);
                                        p->searchExtensionEndCount --;
                                }
                        }
                }
                extension_one_char (t, src);
        }
}

static void
in_script_or_style (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        /* Allocate memory to store the script or style */
        if (p->scriptCodeSize + 11 > p->scriptCodeMaxSize)
                p->scriptCode = g_realloc (p->scriptCode, p->scriptCodeMaxSize += 1024);
                        
        if ((**src == '>' ) && ( p->searchFor [p->searchCount] == '>')) {
                (*src)++;
                p->scriptCode [p->scriptCodeSize] = 0;
                p->scriptCode [p->scriptCodeSize + 1] = 0;
                if (p->script) {
                        p->script = FALSE;
                }
                else {
                        p->style = FALSE;
                }
                g_free (p->scriptCode);
                p->scriptCode = NULL;
        }
        /* Check if a </script> tag is on its way */
        else if (p->searchCount > 0) {
                if (tolower (**src) == p->searchFor [p->searchCount]) {
                        p->searchBuffer [p->searchCount] = **src;
                        p->searchCount++;
                        (*src)++;
                }
                else {
                        gchar *c;

                        p->searchBuffer [p->searchCount] = 0;
                        c = p->searchBuffer;
                        while (*c)
                                p->scriptCode [p->scriptCodeSize++] = *c++;
                        p->scriptCode [p->scriptCodeSize] = **src; (*src)++;
                        p->searchCount = 0;
                }
        }
        else if (**src == '<') {
                p->searchCount = 1;
                p->searchBuffer [0] = '<';
                (*src)++;
        }
        else {
                p->scriptCode [p->scriptCodeSize] = **src;
                (*src)++;
        }
}

static void
add_unichar (HTMLTokenizer *t, gunichar wc)
{
        struct _HTMLTokenizerPrivate *p = t->priv;

        p->utf8_length = 0;

        if (wc != '\0') {
                p->dest += g_unichar_to_utf8 (wc, p->dest);
                *(p->dest) = 0;
        }
        /* g_assert (g_utf8_validate (p->buffer, p->dest - p->buffer, NULL)); */
}

static void
add_byte (HTMLTokenizer *t, const gchar **src)
{
        gunichar wc;
        struct _HTMLTokenizerPrivate *p = t->priv;

        if (p->utf8) {
                p->utf8_buffer[p->utf8_length] = **src;
                p->utf8_length++;

                wc = g_utf8_get_char_validated ((const gchar *)p->utf8_buffer, p->utf8_length);
                if (wc == -1 || p->utf8_length >= (sizeof(p->utf8_buffer)/sizeof(p->utf8_buffer[0]))) {
                        add_unichar (t, '?');
                        (*src)++;
                        return;
                } else if (wc == -2) {
                        /* incomplete character check again */
                        (*src)++;
                        return;
                }
        } else {
                wc = (guchar)**src;
        }

        add_unichar (t, wc);
        (*src)++;
}

static void
flush_entity (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        /* ignore the TAG_ESCAPE when flushing */
        const char *str = p->searchBuffer + 1; 

         while (p->searchCount--) {
                add_byte (t, &str);
        }
}

static void
in_entity (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        gunichar entityValue = 0;

        /* See http://www.mozilla.org/newlayout/testcases/layout/entities.html for a complete entity list,
           ftp://ftp.unicode.org/Public/MAPPINGS/ISO8859/8859-1.TXT
           (or 'man iso_8859_1') for the character encodings. */

        p->searchBuffer [p->searchCount + 1] = **src;
        p->searchBuffer [p->searchCount + 2] = '\0';
                        
        /* Check for &#0000 sequence */
        if (p->searchBuffer[2] == '#') {
                if ((p->searchCount > 1) &&
                    (!isdigit (**src)) &&
                    (p->searchBuffer[3] != 'x')) {
                        /* &#123 */
                        p->searchBuffer [p->searchCount + 1] = '\0';
                        entityValue = strtoul (&(p->searchBuffer [3]),
                                               NULL, 10);
                        p->charEntity = FALSE;
                }
                if ((p->searchCount > 1) &&
                    (!isalnum (**src)) && 
                    (p->searchBuffer[3] == 'x')) {
                        /* &x12AB */
                        p->searchBuffer [p->searchCount + 1] = '\0';
                        entityValue = strtoul (&(p->searchBuffer [4]),
                                               NULL, 16);
                        p->charEntity = FALSE;
                }
        }
        else {
                /* Check for &abc12 sequence */
                if (!isalnum (**src)) {
                        p->charEntity = FALSE;
                        if ((p->searchBuffer [p->searchCount + 1] == ';') ||
                            (!p->tag)) {
                                char *ename = p->searchBuffer + 2;
                                                
                                p->searchBuffer [p->searchCount + 1] = '\0'; /* FIXME sucks */
                                entityValue = html_entity_parse (ename, 0);
                        }
                }
                                
        }

        if (p->searchCount > 9) {
                /* Ignore this sequence since it's too long */
                p->charEntity = FALSE;
                flush_entity (t);
        }
        else if (p->charEntity) {
                                /* Keep searching for end of character entity */
                p->searchCount++;
                (*src)++;
        }
        else {
                /*
                 * my reading of http://www.w3.org/TR/html4/intro/sgmltut.html#h-3.2.2 makes
                 * seem correct to always collapse entity references, even in element names
                 * and attributes. 
                 */
                if (entityValue) {
                        /* Insert plain char */
                        if (entityValue != TAG_ESCAPE)
                                add_unichar (t, entityValue);
                        if (**src == ';')
                                (*src)++;
                }
                else {
                        /* Ignore the sequence, just add it as plaintext */
                        flush_entity (t);
                }
        }
}

static void
in_tag (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;

        p->startTag = FALSE;
        if (**src == '/') {
                if (p->pending == LFPending) {
                        p->pending = NonePending;
                }
        }
        else if (((**src >= 'a') && (**src <= 'z'))
                 || ((**src >= 'A') && (**src <= 'Z'))) {
                                /* Start of a start tag */
        }
        else if (**src == '!') {
                                /* <!-- comment --> */
        }
        else if (**src == '?') {
                                /* <? meta ?> */
        }
        else {
                                /* Invalid tag, just add it */
                if (p->pending)
                        html_tokenizer_add_pending (t);
                add_unichar (t, '<');
                add_byte (t, src);
                return;
        }
                        
        if (p->pending)
                html_tokenizer_add_pending (t);

        if (p->dest > p->buffer) {
                html_tokenizer_append_token (t, p->buffer, p->dest - p->buffer);
                p->dest = p->buffer;
        }
        add_unichar (t, TAG_ESCAPE);
        add_unichar (t, '<');
        p->tag = TRUE;
        p->searchCount = 1; /* Look for <!-- to start comment */
}

static void
start_entity (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;

        (*src)++;
                        
        p->discard = NoneDiscard;
                        
        if (p->pending)
                html_tokenizer_add_pending (t);

        p->charEntity      = TRUE;
        p->searchBuffer[0] = TAG_ESCAPE;
        p->searchBuffer[1] = '&';
        p->searchCount     = 1;
}

static void
start_tag (HTMLTokenizer *t, const gchar **src)
{
        (*src)++;
        t->priv->startTag = TRUE;
        t->priv->discard  = NoneDiscard;
}

static void
end_tag (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        gchar *ptr;

        p->searchCount = 0; /* Stop looking for <!-- sequence */

        add_unichar (t, '>');
        
        /* Make the tag lower case */
        ptr = p->buffer + 2;
        if (p->pre || *ptr == '/') {
                /* End tag */
                p->discard = NoneDiscard;
        }
        else {
                /* Start tag */
                /* Ignore CRLFs after a start tag */
                p->discard = LFDiscard;
        }

        while (*ptr && *ptr !=' ') {
                *ptr = tolower (*ptr);
                ptr++;
        }
        html_tokenizer_append_token (t, p->buffer, p->dest - p->buffer);
        p->dest = p->buffer;
                        
        p->tag = FALSE;
        p->pending = NonePending;
        (*src)++;
                        
        if (strncmp (p->buffer + 2, "pre", 3) == 0) {
                p->pre++;
        }
        else if (strncmp (p->buffer + 2, "/pre", 4) == 0) {
                p->pre--;
        }
        else if (strncmp (p->buffer + 2, "textarea", 8) == 0) {
                p->textarea = TRUE;
        }
        else if (strncmp (p->buffer + 2, "/textarea", 9) == 0) {
                p->textarea = FALSE;
        }
        else if (strncmp (p->buffer + 2, "title", 5) == 0) {
                p->title = TRUE;
        }
        else if (strncmp (p->buffer + 2, "/title", 6) == 0) {
                p->title = FALSE;
        }
        else if (strncmp (p->buffer + 2, "script", 6) == 0) {
                p->script = TRUE;
                p->searchCount = 0;
                p->searchFor = scriptEnd;
                p->scriptCode = g_malloc (1024);
                p->scriptCodeSize = 0;
                p->scriptCodeMaxSize = 1024;
        }
        else if (strncmp (p->buffer + 2, "style", 5) == 0) {
                p->style = TRUE;
                p->searchCount = 0;
                p->searchFor = styleEnd;
                p->scriptCode = g_malloc (1024);
                p->scriptCodeSize = 0;
                p->scriptCodeMaxSize = 1024;
        }
        else if (strncmp (p->buffer + 2, "select", 6) == 0) {
                p->select = TRUE;
        }
        else if (strncmp (p->buffer + 2, "/select", 7) == 0) {
                p->select = FALSE;
        }
        else if (strncmp (p->buffer + 2, "cell", 4) == 0) {
                g_warning ("<cell> tag not supported");
        }
        else if (strncmp (p->buffer + 2, "table", 5) == 0) {
                html_tokenizer_blocking_push (t, Table);
        }
        else {
                if (p->blocking) {
                        const gchar *bn = html_tokenizer_blocking_get_name (t);

                        if (strncmp (p->buffer + 1, bn, strlen (bn)) == 0) {
                                html_tokenizer_blocking_pop (t);
                        }
                }
        }
}

static void
in_crlf (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;

        if (p->tquote) {
                if (p->discard == NoneDiscard)
                        p->pending = SpacePending;
        }
        else if (p->tag) {
                p->searchCount = 0; /* Stop looking for <!-- sequence */
                if (p->discard == NoneDiscard)
                        p->pending = SpacePending; /* Treat LFs inside tags as spaces */
        }
        else if (p->pre || p->textarea) {
                if (p->discard == LFDiscard) {
                        /* Ignore this LF */
                        p->discard = NoneDiscard; /*  We have discarded 1 LF */
                } else {
                        /* Process this LF */
                        if (p->pending)
                                html_tokenizer_add_pending (t);
                        p->pending = LFPending;
                }
        }
        else {
                if (p->discard == LFDiscard) {
                        /* Ignore this LF */
                        p->discard = NoneDiscard; /* We have discarded 1 LF */
                } else {
                        /* Process this LF */
                        if (p->pending == NonePending)
                                p->pending = LFPending;
                }
        }
        /* Check for MS-DOS CRLF sequence */
        if (**src == '\r') {
                p->skipLF = TRUE;
        }
        (*src)++;
}

static void
in_space_or_tab (HTMLTokenizer *t, const gchar **src)
{
        if (t->priv->tquote) {
                if (t->priv->discard == NoneDiscard)
                        t->priv->pending = SpacePending;
        }
        else if (t->priv->tag) {
                t->priv->searchCount = 0; /* Stop looking for <!-- sequence */
                if (t->priv->discard == NoneDiscard)
                        t->priv->pending = SpacePending;
        }
        else if (t->priv->pre || t->priv->textarea) {
                if (t->priv->pending)
                        html_tokenizer_add_pending (t);
                if (**src == ' ')
                        t->priv->pending = SpacePending;
                else
                        t->priv->pending = TabPending;
        }
        else {
                t->priv->pending = SpacePending;
        }
        (*src)++;
}

static void
in_quoted (HTMLTokenizer *t, const gchar **src)
{
        /* We treat ' and " the same in tags " */
        t->priv->discard = NoneDiscard;
        if (t->priv->tag) {
                t->priv->searchCount = 0; /* Stop looking for <!-- sequence */
                if ((t->priv->tquote == SINGLE_QUOTE && **src == '\"') /* match " */
                    || (t->priv->tquote == DOUBLE_QUOTE && **src == '\'')) {
                        add_unichar (t, **src);
                        (*src)++;
                } else if (*(t->priv->dest-1) == '=' && !t->priv->tquote) {
                        t->priv->discard = SpaceDiscard;
                        t->priv->pending = NonePending;
                                        
                        if (**src == '\"') /* match " */
                                t->priv->tquote = DOUBLE_QUOTE;
                        else
                                t->priv->tquote = SINGLE_QUOTE;
                        add_unichar (t, **src);
                        (*src)++;
                }
                else if (t->priv->tquote) {
                        t->priv->tquote = NO_QUOTE;
                        add_byte (t, src);
                        t->priv->pending = SpacePending;
                }
                else {
                        /* Ignore stray "\'" */
                        (*src)++;
                }
        }
        else {
                if (t->priv->pending)
                        html_tokenizer_add_pending (t);

                add_byte (t, src);
        }
}

static void
in_assignment (HTMLTokenizer *t, const gchar **src)
{
        t->priv->discard = NoneDiscard;
        if (t->priv->tag) {
                t->priv->searchCount = 0; /* Stop looking for <!-- sequence */
                add_unichar (t, '=');
                if (!t->priv->tquote) {
                        t->priv->pending = NonePending;
                        t->priv->discard = SpaceDiscard;
                }
        }
        else {
                if (t->priv->pending)
                        html_tokenizer_add_pending (t);

                add_unichar (t, '=');
        }
        (*src)++;
}

inline static void
in_plain (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        p->discard = NoneDiscard;
        if (p->pending)
                html_tokenizer_add_pending (t);
                        
        if (p->tag) {
                if (p->searchCount > 0) {
                        if (**src == commentStart[p->searchCount]) {
                                p->searchCount++;
                                if (p->searchCount == 4) {
                                        /* Found <!-- sequence */
                                        p->comment = TRUE;
                                        p->dest = p->buffer;
                                        p->tag = FALSE;
                                        p->searchCount = 0;
                                        return;
                                }
                        }
                        else {
                                p->searchCount = 0; /* Stop lookinf for <!-- sequence */
                        }
                }
        }

        add_byte (t, src);
}

static void
html_tokenizer_tokenize_one_char (HTMLTokenizer *t, const gchar **src)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        prepare_enough_space (t);

        if (p->skipLF && **src != '\n')
                p->skipLF = FALSE;

        if (p->skipLF)
                (*src) ++;
        else if (p->comment)
                in_comment (t, src);
        else if (p->extension)
                in_extension (t, src);
        else if (p->script || p->style)
                in_script_or_style (t, src);
        else if (p->charEntity)
                in_entity (t, src);
        else if (p->startTag)
                in_tag (t, src);
        else if (**src == '&')
                start_entity (t, src);
        else if (**src == '<' && !p->tag)
                start_tag (t, src);
        else if (**src == '>' && p->tag && !p->tquote)
                end_tag (t, src);
        else if ((**src == '\n') || (**src == '\r'))
                in_crlf (t, src);
        else if ((**src == ' ') || (**src == '\t'))
                in_space_or_tab (t, src);
        else if (**src == '\"' || **src == '\'') /* match " ' */
                in_quoted (t, src);
        else if (**src == '=')
                in_assignment (t, src);
        else
                in_plain (t, src);
}

static void
html_tokenizer_real_write (HTMLTokenizer *t, const gchar *string, size_t size)
{
        const gchar *src = string;

        while ((src - string) < size)
                html_tokenizer_tokenize_one_char (t, &src);
}

static gchar *
html_tokenizer_blocking_get_name (HTMLTokenizer *t)
{
        switch (GPOINTER_TO_INT (t->priv->blocking->data)) {
        case Table:
                return "</table";
        }
        
        return "";
}

static void
html_tokenizer_blocking_push (HTMLTokenizer *t, HTMLTokenType tt)
{
        struct _HTMLTokenizerPrivate *p = t->priv;
        
        /* block tokenizer - we must block last token in buffers as it was already added */
        if (!p->blocking) {
                p->tokens_num--;
                p->blocking_tokens_num++;
        }
        p->blocking = g_list_prepend (p->blocking, GINT_TO_POINTER (tt));
}

static void
html_tokenizer_blocking_pop (HTMLTokenizer *t)
{
        struct _HTMLTokenizerPrivate *p = t->priv;

        p->blocking = g_list_remove (p->blocking, p->blocking->data);

        /* unblock tokenizer */
        if (!p->blocking) {
                p->tokens_num += p->blocking_tokens_num;
                p->blocking_tokens_num = 0;
        }
}

/** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **/

void
html_tokenizer_begin (HTMLTokenizer *t, gchar *content_type)
{
        g_return_if_fail (t && HTML_IS_TOKENIZER (t));

        gtk_signal_emit (GTK_OBJECT (t),
                         html_tokenizer_signals[HTML_TOKENIZER_BEGIN_SIGNAL],
                         content_type);
}

void
html_tokenizer_end (HTMLTokenizer *t)
{
        g_return_if_fail (t && HTML_IS_TOKENIZER (t));

        gtk_signal_emit (GTK_OBJECT (t),
                         html_tokenizer_signals[HTML_TOKENIZER_END_SIGNAL]);
}

void
html_tokenizer_write (HTMLTokenizer *t, const gchar *str, size_t size)
{
        HTMLTokenizerClass *klass;

        g_return_if_fail (t && HTML_IS_TOKENIZER (t));
        klass = HTML_TOKENIZER_CLASS (GTK_OBJECT (t)->klass);
        
        if (klass->write)
                klass->write (t, str, size);
        else
                g_warning ("No write method defined.");
}

gchar *
html_tokenizer_peek_token (HTMLTokenizer *t)
{
        HTMLTokenizerClass *klass;

        g_return_val_if_fail (t && HTML_IS_TOKENIZER (t), NULL);

        klass = HTML_TOKENIZER_CLASS (GTK_OBJECT (t)->klass);
        
        if (klass->peek_token)
                return klass->peek_token (t);
        
        g_warning ("No peek_token method defined.");
        return NULL;

}

gchar *
html_tokenizer_next_token (HTMLTokenizer *t)
{
        HTMLTokenizerClass *klass;

        g_return_val_if_fail (t && HTML_IS_TOKENIZER (t), NULL);

        klass = HTML_TOKENIZER_CLASS (GTK_OBJECT (t)->klass);
        
        if (klass->next_token)
                return klass->next_token (t);

        g_warning ("No next_token method defined.");
        return NULL;
}

gboolean
html_tokenizer_has_more_tokens (HTMLTokenizer *t)
{
        HTMLTokenizerClass *klass;

        g_return_val_if_fail (t && HTML_IS_TOKENIZER (t), FALSE);

        klass = HTML_TOKENIZER_CLASS (GTK_OBJECT (t)->klass);
        
        if (klass->has_more) {
                return klass->has_more (t);
        }

        g_warning ("No has_more method defined.");
        return FALSE;
        
}

HTMLTokenizer *
html_tokenizer_clone (HTMLTokenizer *t)
{
        HTMLTokenizerClass *klass;
        
        if (t == NULL)
                return NULL;
        g_return_val_if_fail (HTML_IS_TOKENIZER (t), NULL);

        klass = HTML_TOKENIZER_CLASS (GTK_OBJECT (t)->klass);
        
        if (klass->clone)
                return klass->clone (t);
        
        g_warning ("No clone method defined.");
        return NULL;
}