git.maemo.org Git - dh-make-perl/blob - dev/arm/libhtml-parser-perl/libhtml-parser-perl-3.56/hparser.c

   1 /* $Id: hparser.c,v 2.134 2007/01/12 10:54:06 gisle Exp $
   2  *
   3  * Copyright 1999-2007, Gisle Aas
   4  * Copyright 1999-2000, Michael A. Chase
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the same terms as Perl itself.
   8  */
   9
  10 #ifndef EXTERN
  11 #define EXTERN extern
  12 #endif
  13
  14 #include "hctype.h"    /* isH...() macros */
  15 #include "tokenpos.h"  /* dTOKEN; PUSH_TOKEN() */
  16
  17
  18 static
  19 struct literal_tag {
  20     int len;
  21     char* str;
  22     int is_cdata;
  23 }
  24 literal_mode_elem[] =
  25 {
  26     {6, "script", 1},
  27     {5, "style", 1},
  28     {3, "xmp", 1},
  29     {9, "plaintext", 1},
  30     {5, "title", 0},
  31     {8, "textarea", 0},
  32     {0, 0, 0}
  33 };
  34
  35 enum argcode {
  36     ARG_SELF = 1,  /* need to avoid '\0' in argspec string */
  37     ARG_TOKENS,
  38     ARG_TOKENPOS,
  39     ARG_TOKEN0,
  40     ARG_TAGNAME,
  41     ARG_TAG,
  42     ARG_ATTR,
  43     ARG_ATTRARR,
  44     ARG_ATTRSEQ,
  45     ARG_TEXT,
  46     ARG_DTEXT,
  47     ARG_IS_CDATA,
  48     ARG_SKIPPED_TEXT,
  49     ARG_OFFSET,
  50     ARG_OFFSET_END,
  51     ARG_LENGTH,
  52     ARG_LINE,
  53     ARG_COLUMN,
  54     ARG_EVENT,
  55     ARG_UNDEF,
  56     ARG_LITERAL, /* Always keep last */
  57
  58     /* extra flags always encoded first */
  59     ARG_FLAG_FLAT_ARRAY
  60 };
  61
  62 char *argname[] = {
  63     /* Must be in the same order as enum argcode */
  64     "self",     /* ARG_SELF */
  65     "tokens",   /* ARG_TOKENS */
  66     "tokenpos", /* ARG_TOKENPOS */
  67     "token0",   /* ARG_TOKEN0 */
  68     "tagname",  /* ARG_TAGNAME */
  69     "tag",      /* ARG_TAG */
  70     "attr",     /* ARG_ATTR */
  71     "@attr",    /* ARG_ATTRARR */
  72     "attrseq",  /* ARG_ATTRSEQ */
  73     "text",     /* ARG_TEXT */
  74     "dtext",    /* ARG_DTEXT */
  75     "is_cdata", /* ARG_IS_CDATA */
  76     "skipped_text", /* ARG_SKIPPED_TEXT */
  77     "offset",   /* ARG_OFFSET */
  78     "offset_end", /* ARG_OFFSET_END */
  79     "length",   /* ARG_LENGTH */
  80     "line",     /* ARG_LINE */
  81     "column",   /* ARG_COLUMN */
  82     "event",    /* ARG_EVENT */
  83     "undef",    /* ARG_UNDEF */
  84     /* ARG_LITERAL (not compared) */
  85     /* ARG_FLAG_FLAT_ARRAY */
  86 };
  87
  88 #define CASE_SENSITIVE(p_state) \
  89          ((p_state)->xml_mode || (p_state)->case_sensitive)
  90 #define STRICT_NAMES(p_state) \
  91          ((p_state)->xml_mode || (p_state)->strict_names)
  92 #define ALLOW_EMPTY_TAG(p_state) \
  93          ((p_state)->xml_mode || (p_state)->empty_element_tags)
  94
  95 static void flush_pending_text(PSTATE* p_state, SV* self);
  96
  97 /*
  98  * Parser functions.
  99  *
 100  *   parse()                       - top level entry point.
 101  *                                   deals with text and calls one of its
 102  *                                   subordinate parse_*() routines after
 103  *                                   looking at the first char after "<"
 104  *     parse_decl()                - deals with declarations         <!...>
 105  *       parse_comment()           - deals with <!-- ... -->
 106  *       parse_marked_section      - deals with <![ ... [ ... ]]>
 107  *     parse_end()                 - deals with end tags             </...>
 108  *     parse_start()               - deals with start tags           <A...>
 109  *     parse_process()             - deals with process instructions <?...>
 110  *     parse_null()                - deals with anything else        <....>
 111  *
 112  *     report_event() - called whenever any of the parse*() routines
 113  *                      has recongnized something.
 114  */
 115
 116 static void
 117 report_event(PSTATE* p_state,
 118              event_id_t event,
 119              char *beg, char *end, U32 utf8,
 120              token_pos_t *tokens, int num_tokens,
 121              SV* self
 122             )
 123 {
 124     struct p_handler *h;
 125     dTHX;
 126     dSP;
 127     AV *array;
 128     STRLEN my_na;
 129     char *argspec;
 130     char *s;
 131     STRLEN offset;
 132     STRLEN line;
 133     STRLEN column;
 134
 135 #ifdef UNICODE_HTML_PARSER
 136     #define CHR_DIST(a,b) (utf8 ? utf8_distance((U8*)(a),(U8*)(b)) : (a) - (b))
 137 #else
 138     #define CHR_DIST(a,b) ((a) - (b))
 139 #endif
 140
 141     /* some events might still fire after a handler has signaled eof
 142      * so suppress them here.
 143      */
 144     if (p_state->eof)
 145         return;
 146
 147     /* capture offsets */
 148     offset = p_state->offset;
 149     line = p_state->line;
 150     column = p_state->column;
 151
 152 #if 0
 153     {  /* used for debugging at some point */
 154         char *s = beg;
 155         int i;
 156
 157         /* print debug output */
 158         switch(event) {
 159         case E_DECLARATION: printf("DECLARATION"); break;
 160         case E_COMMENT:     printf("COMMENT"); break;
 161         case E_START:       printf("START"); break;
 162         case E_END:         printf("END"); break;
 163         case E_TEXT:        printf("TEXT"); break;
 164         case E_PROCESS:     printf("PROCESS"); break;
 165         case E_NONE:        printf("NONE"); break;
 166         default:            printf("EVENT #%d", event); break;
 167         }
 168
 169         printf(" [");
 170         while (s < end) {
 171             if (*s == '\n') {
 172                 putchar('\\'); putchar('n');
 173             }
 174             else
 175                 putchar(*s);
 176             s++;
 177         }
 178         printf("] %d\n", end - beg);
 179         for (i = 0; i < num_tokens; i++) {
 180             printf("  token %d: %d %d\n",
 181                    i,
 182                    tokens[i].beg - beg,
 183                    tokens[i].end - tokens[i].beg);
 184         }
 185     }
 186 #endif
 187
 188     if (p_state->pending_end_tag && event != E_TEXT && event != E_COMMENT) {
 189         token_pos_t t;
 190         char dummy;
 191         t.beg = p_state->pending_end_tag;
 192         t.end = p_state->pending_end_tag + strlen(p_state->pending_end_tag);
 193         p_state->pending_end_tag = 0;
 194         report_event(p_state, E_END, &dummy, &dummy, 0, &t, 1, self);
 195         SPAGAIN;
 196     }
 197
 198     /* update offsets */
 199     p_state->offset += CHR_DIST(end, beg);
 200     if (line) {
 201         char *s = beg;
 202         char *nl = NULL;
 203         while (s < end) {
 204             if (*s == '\n') {
 205                 p_state->line++;
 206                 nl = s;
 207             }
 208             s++;
 209         }
 210         if (nl)
 211             p_state->column = CHR_DIST(end, nl) - 1;
 212         else
 213             p_state->column += CHR_DIST(end, beg);
 214     }
 215
 216     if (event == E_NONE)
 217         goto IGNORE_EVENT;
 218
 219 #ifdef MARKED_SECTION
 220     if (p_state->ms == MS_IGNORE)
 221         goto IGNORE_EVENT;
 222 #endif
 223
 224     /* tag filters */
 225     if (p_state->ignore_tags || p_state->report_tags || p_state->ignore_elements) {
 226
 227         if (event == E_START || event == E_END) {
 228             SV* tagname = p_state->tmp;
 229
 230             assert(num_tokens >= 1);
 231             sv_setpvn(tagname, tokens[0].beg, tokens[0].end - tokens[0].beg);
 232             if (utf8)
 233                 SvUTF8_on(tagname);
 234             else
 235                 SvUTF8_off(tagname);
 236             if (!CASE_SENSITIVE(p_state))
 237                 sv_lower(aTHX_ tagname);
 238
 239             if (p_state->ignoring_element) {
 240                 if (sv_eq(p_state->ignoring_element, tagname)) {
 241                     if (event == E_START)
 242                         p_state->ignore_depth++;
 243                     else if (--p_state->ignore_depth == 0) {
 244                         SvREFCNT_dec(p_state->ignoring_element);
 245                         p_state->ignoring_element = 0;
 246                     }
 247                 }
 248                 goto IGNORE_EVENT;
 249             }
 250
 251             if (p_state->ignore_elements &&
 252                 hv_fetch_ent(p_state->ignore_elements, tagname, 0, 0))
 253             {
 254                 if (event == E_START) {
 255                     p_state->ignoring_element = newSVsv(tagname);
 256                     p_state->ignore_depth = 1;
 257                 }
 258                 goto IGNORE_EVENT;
 259             }
 260
 261             if (p_state->ignore_tags &&
 262                 hv_fetch_ent(p_state->ignore_tags, tagname, 0, 0))
 263             {
 264                 goto IGNORE_EVENT;
 265             }
 266             if (p_state->report_tags &&
 267                 !hv_fetch_ent(p_state->report_tags, tagname, 0, 0))
 268             {
 269                 goto IGNORE_EVENT;
 270             }
 271         }
 272         else if (p_state->ignoring_element) {
 273             goto IGNORE_EVENT;
 274         }
 275     }
 276
 277     h = &p_state->handlers[event];
 278     if (!h->cb) {
 279         /* event = E_DEFAULT; */
 280         h = &p_state->handlers[E_DEFAULT];
 281         if (!h->cb)
 282             goto IGNORE_EVENT;
 283     }
 284
 285     if (SvTYPE(h->cb) != SVt_PVAV && !SvTRUE(h->cb)) {
 286         /* FALSE scalar ('' or 0) means IGNORE this event */
 287         return;
 288     }
 289
 290     if (p_state->unbroken_text && event == E_TEXT) {
 291         /* should buffer text */
 292         if (!p_state->pend_text)
 293             p_state->pend_text = newSV(256);
 294         if (SvOK(p_state->pend_text)) {
 295             if (p_state->is_cdata != p_state->pend_text_is_cdata) {
 296                 flush_pending_text(p_state, self);
 297                 SPAGAIN;
 298                 goto INIT_PEND_TEXT;
 299             }
 300         }
 301         else {
 302         INIT_PEND_TEXT:
 303             p_state->pend_text_offset = offset;
 304             p_state->pend_text_line = line;
 305             p_state->pend_text_column = column;
 306             p_state->pend_text_is_cdata = p_state->is_cdata;
 307             sv_setpvn(p_state->pend_text, "", 0);
 308             if (!utf8)
 309                 SvUTF8_off(p_state->pend_text);
 310         }
 311 #ifdef UNICODE_HTML_PARSER
 312         if (utf8 && !SvUTF8(p_state->pend_text))
 313             sv_utf8_upgrade(p_state->pend_text);
 314         if (utf8 || !SvUTF8(p_state->pend_text)) {
 315             sv_catpvn(p_state->pend_text, beg, end - beg);
 316         }
 317         else {
 318             SV *tmp = newSVpvn(beg, end - beg);
 319             sv_utf8_upgrade(tmp);
 320             sv_catsv(p_state->pend_text, tmp);
 321             SvREFCNT_dec(tmp);
 322         }
 323 #else
 324         sv_catpvn(p_state->pend_text, beg, end - beg);
 325 #endif
 326         return;
 327     }
 328     else if (p_state->pend_text && SvOK(p_state->pend_text)) {
 329         flush_pending_text(p_state, self);
 330         SPAGAIN;
 331     }
 332
 333     /* At this point we have decided to generate an event callback */
 334
 335     argspec = h->argspec ? SvPV(h->argspec, my_na) : "";
 336
 337     if (SvTYPE(h->cb) == SVt_PVAV) {
 338
 339         if (*argspec == ARG_FLAG_FLAT_ARRAY) {
 340             argspec++;
 341             array = (AV*)h->cb;
 342         }
 343         else {
 344             /* start sub-array for accumulator array */
 345             array = newAV();
 346         }
 347     }
 348     else {
 349         array = 0;
 350         if (*argspec == ARG_FLAG_FLAT_ARRAY)
 351             argspec++;
 352
 353         /* start argument stack for callback */
 354         ENTER;
 355         SAVETMPS;
 356         PUSHMARK(SP);
 357     }
 358
 359     for (s = argspec; *s; s++) {
 360         SV* arg = 0;
 361         int push_arg = 1;
 362         enum argcode argcode = (enum argcode)*s;
 363
 364         switch( argcode ) {
 365
 366         case ARG_SELF:
 367             arg = sv_mortalcopy(self);
 368             break;
 369
 370         case ARG_TOKENS:
 371             if (num_tokens >= 1) {
 372                 AV* av = newAV();
 373                 SV* prev_token = &PL_sv_undef;
 374                 int i;
 375                 av_extend(av, num_tokens);
 376                 for (i = 0; i < num_tokens; i++) {
 377                     if (tokens[i].beg) {
 378                         prev_token = newSVpvn(tokens[i].beg, tokens[i].end-tokens[i].beg);
 379                         if (utf8)
 380                             SvUTF8_on(prev_token);
 381                         av_push(av, prev_token);
 382                     }
 383                     else { /* boolean */
 384                         av_push(av, p_state->bool_attr_val
 385                                 ? newSVsv(p_state->bool_attr_val)
 386                                 : newSVsv(prev_token));
 387                     }
 388                 }
 389                 arg = sv_2mortal(newRV_noinc((SV*)av));
 390             }
 391             break;
 392
 393         case ARG_TOKENPOS:
 394             if (num_tokens >= 1 && tokens[0].beg >= beg) {
 395                 AV* av = newAV();
 396                 int i;
 397                 av_extend(av, num_tokens*2);
 398                 for (i = 0; i < num_tokens; i++) {
 399                     if (tokens[i].beg) {
 400                         av_push(av, newSViv(CHR_DIST(tokens[i].beg, beg)));
 401                         av_push(av, newSViv(CHR_DIST(tokens[i].end, tokens[i].beg)));
 402                     }
 403                     else { /* boolean tag value */
 404                         av_push(av, newSViv(0));
 405                         av_push(av, newSViv(0));
 406                     }
 407                 }
 408                 arg = sv_2mortal(newRV_noinc((SV*)av));
 409             }
 410             break;
 411
 412         case ARG_TOKEN0:
 413         case ARG_TAGNAME:
 414             /* fall through */
 415
 416         case ARG_TAG:
 417             if (num_tokens >= 1) {
 418                 arg = sv_2mortal(newSVpvn(tokens[0].beg,
 419                                           tokens[0].end - tokens[0].beg));
 420                 if (utf8)
 421                     SvUTF8_on(arg);
 422                 if (!CASE_SENSITIVE(p_state) && argcode != ARG_TOKEN0)
 423                     sv_lower(aTHX_ arg);
 424                 if (argcode == ARG_TAG && event != E_START) {
 425                     char *e_type = "!##/#?#";
 426                     sv_insert(arg, 0, 0, &e_type[event], 1);
 427                 }
 428             }
 429             break;
 430
 431         case ARG_ATTR:
 432         case ARG_ATTRARR:
 433             if (event == E_START) {
 434                 HV* hv;
 435                 int i;
 436                 if (argcode == ARG_ATTR) {
 437                     hv = newHV();
 438                     arg = sv_2mortal(newRV_noinc((SV*)hv));
 439                 }
 440                 else {
 441 #ifdef __GNUC__
 442                     /* gcc -Wall reports this variable as possibly used uninitialized */
 443                     hv = 0;
 444 #endif
 445                     push_arg = 0;  /* deal with argument pushing here */
 446                 }
 447
 448                 for (i = 1; i < num_tokens; i += 2) {
 449                     SV* attrname = newSVpvn(tokens[i].beg,
 450                                             tokens[i].end-tokens[i].beg);
 451                     SV* attrval;
 452
 453                     if (utf8)
 454                         SvUTF8_on(attrname);
 455                     if (tokens[i+1].beg) {
 456                         char *beg = tokens[i+1].beg;
 457                         STRLEN len = tokens[i+1].end - beg;
 458                         if (*beg == '"' || *beg == '\'') {
 459                             assert(len >= 2 && *beg == beg[len-1]);
 460                             beg++; len -= 2;
 461                         }
 462                         attrval = newSVpvn(beg, len);
 463                         if (utf8)
 464                             SvUTF8_on(attrval);
 465                         if (!p_state->attr_encoded) {
 466 #ifdef UNICODE_HTML_PARSER
 467                             if (p_state->utf8_mode)
 468                                 sv_utf8_decode(attrval);
 469 #endif
 470                             decode_entities(aTHX_ attrval, p_state->entity2char, 0);
 471                             if (p_state->utf8_mode)
 472                                 SvUTF8_off(attrval);
 473                         }
 474                     }
 475                     else { /* boolean */
 476                         if (p_state->bool_attr_val)
 477                             attrval = newSVsv(p_state->bool_attr_val);
 478                         else
 479                             attrval = newSVsv(attrname);
 480                     }
 481
 482                     if (!CASE_SENSITIVE(p_state))
 483                         sv_lower(aTHX_ attrname);
 484
 485                     if (argcode == ARG_ATTR) {
 486                         if (hv_exists_ent(hv, attrname, 0) ||
 487                             !hv_store_ent(hv, attrname, attrval, 0)) {
 488                             SvREFCNT_dec(attrval);
 489                         }
 490                         SvREFCNT_dec(attrname);
 491                     }
 492                     else { /* ARG_ATTRARR */
 493                         if (array) {
 494                             av_push(array, attrname);
 495                             av_push(array, attrval);
 496                         }
 497                         else {
 498                             XPUSHs(sv_2mortal(attrname));
 499                             XPUSHs(sv_2mortal(attrval));
 500                         }
 501                     }
 502                 }
 503             }
 504             else if (argcode == ARG_ATTRARR) {
 505                 push_arg = 0;
 506             }
 507             break;
 508
 509         case ARG_ATTRSEQ:       /* (v2 compatibility stuff) */
 510             if (event == E_START) {
 511                 AV* av = newAV();
 512                 int i;
 513                 for (i = 1; i < num_tokens; i += 2) {
 514                     SV* attrname = newSVpvn(tokens[i].beg,
 515                                             tokens[i].end-tokens[i].beg);
 516                     if (utf8)
 517                         SvUTF8_on(attrname);
 518                     if (!CASE_SENSITIVE(p_state))
 519                         sv_lower(aTHX_ attrname);
 520                     av_push(av, attrname);
 521                 }
 522                 arg = sv_2mortal(newRV_noinc((SV*)av));
 523             }
 524             break;
 525
 526         case ARG_TEXT:
 527             arg = sv_2mortal(newSVpvn(beg, end - beg));
 528             if (utf8)
 529                 SvUTF8_on(arg);
 530             break;
 531
 532         case ARG_DTEXT:
 533             if (event == E_TEXT) {
 534                 arg = sv_2mortal(newSVpvn(beg, end - beg));
 535                 if (utf8)
 536                     SvUTF8_on(arg);
 537                 if (!p_state->is_cdata) {
 538 #ifdef UNICODE_HTML_PARSER
 539                     if (p_state->utf8_mode)
 540                         sv_utf8_decode(arg);
 541 #endif
 542                     decode_entities(aTHX_ arg, p_state->entity2char, 1);
 543                     if (p_state->utf8_mode)
 544                         SvUTF8_off(arg);
 545                 }
 546             }
 547             break;
 548
 549         case ARG_IS_CDATA:
 550             if (event == E_TEXT) {
 551                 arg = boolSV(p_state->is_cdata);
 552             }
 553             break;
 554
 555         case ARG_SKIPPED_TEXT:
 556             arg = sv_2mortal(p_state->skipped_text);
 557             p_state->skipped_text = newSVpvn("", 0);
 558             break;
 559
 560         case ARG_OFFSET:
 561             arg = sv_2mortal(newSViv(offset));
 562             break;
 563
 564         case ARG_OFFSET_END:
 565             arg = sv_2mortal(newSViv(offset + CHR_DIST(end, beg)));
 566             break;
 567
 568         case ARG_LENGTH:
 569             arg = sv_2mortal(newSViv(CHR_DIST(end, beg)));
 570             break;
 571
 572         case ARG_LINE:
 573             arg = sv_2mortal(newSViv(line));
 574             break;
 575
 576         case ARG_COLUMN:
 577             arg = sv_2mortal(newSViv(column));
 578             break;
 579
 580         case ARG_EVENT:
 581             assert(event >= 0 && event < EVENT_COUNT);
 582             arg = sv_2mortal(newSVpv(event_id_str[event], 0));
 583             break;
 584
 585         case ARG_LITERAL:
 586         {
 587             int len = (unsigned char)s[1];
 588             arg = sv_2mortal(newSVpvn(s+2, len));
 589             if (SvUTF8(h->argspec))
 590                 SvUTF8_on(arg);
 591             s += len + 1;
 592         }
 593         break;
 594
 595         case ARG_UNDEF:
 596             arg = sv_mortalcopy(&PL_sv_undef);
 597             break;
 598
 599         default:
 600             arg = sv_2mortal(newSVpvf("Bad argspec %d", *s));
 601             break;
 602         }
 603
 604         if (push_arg) {
 605             if (!arg)
 606                 arg = sv_mortalcopy(&PL_sv_undef);
 607
 608             if (array) {
 609                 /* have to fix mortality here or add mortality to
 610                  * XPUSHs after removing it from the switch cases.
 611                  */
 612                 av_push(array, SvREFCNT_inc(arg));
 613             }
 614             else {
 615                 XPUSHs(arg);
 616             }
 617         }
 618     }
 619
 620     if (array) {
 621         if (array != (AV*)h->cb)
 622             av_push((AV*)h->cb, newRV_noinc((SV*)array));
 623     }
 624     else {
 625         PUTBACK;
 626
 627         if ((enum argcode)*argspec == ARG_SELF && !SvROK(h->cb)) {
 628             char *method = SvPV(h->cb, my_na);
 629             perl_call_method(method, G_DISCARD | G_EVAL | G_VOID);
 630         }
 631         else {
 632             perl_call_sv(h->cb, G_DISCARD | G_EVAL | G_VOID);
 633         }
 634
 635         if (SvTRUE(ERRSV)) {
 636             RETHROW;
 637         }
 638
 639         FREETMPS;
 640         LEAVE;
 641     }
 642     if (p_state->skipped_text)
 643         SvCUR_set(p_state->skipped_text, 0);
 644     return;
 645
 646 IGNORE_EVENT:
 647     if (p_state->skipped_text) {
 648         if (event != E_TEXT && p_state->pend_text && SvOK(p_state->pend_text))
 649             flush_pending_text(p_state, self);
 650 #ifdef UNICODE_HTML_PARSER
 651         if (utf8 && !SvUTF8(p_state->skipped_text))
 652             sv_utf8_upgrade(p_state->skipped_text);
 653         if (utf8 || !SvUTF8(p_state->skipped_text)) {
 654 #endif
 655             sv_catpvn(p_state->skipped_text, beg, end - beg);
 656 #ifdef UNICODE_HTML_PARSER
 657         }
 658         else {
 659             SV *tmp = newSVpvn(beg, end - beg);
 660             sv_utf8_upgrade(tmp);
 661             sv_catsv(p_state->pend_text, tmp);
 662             SvREFCNT_dec(tmp);
 663         }
 664 #endif
 665     }
 666 #undef CHR_DIST
 667     return;
 668 }
 669
 670
 671 EXTERN SV*
 672 argspec_compile(SV* src, PSTATE* p_state)
 673 {
 674     dTHX;
 675     SV* argspec = newSVpvn("", 0);
 676     STRLEN len;
 677     char *s = SvPV(src, len);
 678     char *end = s + len;
 679
 680     if (SvUTF8(src))
 681         SvUTF8_on(argspec);
 682
 683     while (isHSPACE(*s))
 684         s++;
 685
 686     if (*s == '@') {
 687         /* try to deal with '@{ ... }' wrapping */
 688         char *tmp = s + 1;
 689         while (isHSPACE(*tmp))
 690             tmp++;
 691         if (*tmp == '{') {
 692             char c = ARG_FLAG_FLAT_ARRAY;
 693             sv_catpvn(argspec, &c, 1);
 694             tmp++;
 695             while (isHSPACE(*tmp))
 696                 tmp++;
 697             s = tmp;
 698         }
 699     }
 700     while (s < end) {
 701         if (isHNAME_FIRST(*s) || *s == '@') {
 702             char *name = s;
 703             int a = ARG_SELF;
 704             char **arg_name;
 705
 706             s++;
 707             while (isHNAME_CHAR(*s))
 708                 s++;
 709
 710             /* check identifier */
 711             for ( arg_name = argname; a < ARG_LITERAL ; ++a, ++arg_name ) {
 712                 if (strnEQ(*arg_name, name, s - name) &&
 713                     (*arg_name)[s - name] == '\0')
 714                     break;
 715             }
 716             if (a < ARG_LITERAL) {
 717                 char c = (unsigned char) a;
 718                 sv_catpvn(argspec, &c, 1);
 719
 720                 if (a == ARG_LINE || a == ARG_COLUMN) {
 721                     if (!p_state->line)
 722                         p_state->line = 1; /* enable tracing of line/column */
 723                 }
 724                 if (a == ARG_SKIPPED_TEXT) {
 725                     if (!p_state->skipped_text) {
 726                         p_state->skipped_text = newSVpvn("", 0);
 727                     }
 728                 }
 729                 if (a == ARG_ATTR || a == ARG_ATTRARR || a == ARG_DTEXT) {
 730                     p_state->argspec_entity_decode++;
 731                 }
 732             }
 733             else {
 734                 croak("Unrecognized identifier %.*s in argspec", s - name, name);
 735             }
 736         }
 737         else if (*s == '"' || *s == '\'') {
 738             char *string_beg = s;
 739             s++;
 740             while (s < end && *s != *string_beg && *s != '\\')
 741                 s++;
 742             if (*s == *string_beg) {
 743                 /* literal */
 744                 int len = s - string_beg - 1;
 745                 unsigned char buf[2];
 746                 if (len > 255)
 747                     croak("Literal string is longer than 255 chars in argspec");
 748                 buf[0] = ARG_LITERAL;
 749                 buf[1] = len;
 750                 sv_catpvn(argspec, (char*)buf, 2);
 751                 sv_catpvn(argspec, string_beg+1, len);
 752                 s++;
 753             }
 754             else if (*s == '\\') {
 755                 croak("Backslash reserved for literal string in argspec");
 756             }
 757             else {
 758                 croak("Unterminated literal string in argspec");
 759             }
 760         }
 761         else {
 762             croak("Bad argspec (%s)", s);
 763         }
 764
 765         while (isHSPACE(*s))
 766             s++;
 767
 768         if (*s == '}' && SvPVX(argspec)[0] == ARG_FLAG_FLAT_ARRAY) {
 769             /* end of '@{ ... }' */
 770             s++;
 771             while (isHSPACE(*s))
 772                 s++;
 773             if (s < end)
 774                 croak("Bad argspec: stuff after @{...} (%s)", s);
 775         }
 776
 777         if (s == end)
 778             break;
 779         if (*s != ',') {
 780             croak("Missing comma separator in argspec");
 781         }
 782         s++;
 783         while (isHSPACE(*s))
 784             s++;
 785     }
 786     return argspec;
 787 }
 788
 789
 790 static void
 791 flush_pending_text(PSTATE* p_state, SV* self)
 792 {
 793     dTHX;
 794     bool   old_unbroken_text = p_state->unbroken_text;
 795     SV*    old_pend_text     = p_state->pend_text;
 796     bool   old_is_cdata      = p_state->is_cdata;
 797     STRLEN old_offset        = p_state->offset;
 798     STRLEN old_line          = p_state->line;
 799     STRLEN old_column        = p_state->column;
 800
 801     assert(p_state->pend_text && SvOK(p_state->pend_text));
 802
 803     p_state->unbroken_text = 0;
 804     p_state->pend_text     = 0;
 805     p_state->is_cdata      = p_state->pend_text_is_cdata;
 806     p_state->offset        = p_state->pend_text_offset;
 807     p_state->line          = p_state->pend_text_line;
 808     p_state->column        = p_state->pend_text_column;
 809
 810     report_event(p_state, E_TEXT,
 811                  SvPVX(old_pend_text), SvEND(old_pend_text),
 812                  SvUTF8(old_pend_text), 0, 0, self);
 813     SvOK_off(old_pend_text);
 814
 815     p_state->unbroken_text = old_unbroken_text;
 816     p_state->pend_text     = old_pend_text;
 817     p_state->is_cdata      = old_is_cdata;
 818     p_state->offset        = old_offset;
 819     p_state->line          = old_line;
 820     p_state->column        = old_column;
 821 }
 822
 823 static char*
 824 skip_until_gt(char *beg, char *end)
 825 {
 826     /* tries to emulate quote skipping behaviour observed in MSIE */
 827     char *s = beg;
 828     char quote = '\0';
 829     char prev = ' ';
 830     while (s < end) {
 831         if (!quote && *s == '>')
 832             return s;
 833         if (*s == '"' || *s == '\'') {
 834             if (*s == quote) {
 835                 quote = '\0';  /* end of quoted string */
 836             }
 837             else if (!quote && (prev == ' ' || prev == '=')) {
 838                 quote = *s;
 839             }
 840         }
 841         prev = *s++;
 842     }
 843     return end;
 844 }
 845
 846 static char*
 847 parse_comment(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
 848 {
 849     char *s = beg;
 850
 851     if (p_state->strict_comment) {
 852         dTOKENS(4);
 853         char *start_com = s;  /* also used to signal inside/outside */
 854
 855         while (1) {
 856             /* try to locate "--" */
 857         FIND_DASH_DASH:
 858             /* printf("find_dash_dash: [%s]\n", s); */
 859             while (s < end && *s != '-' && *s != '>')
 860                 s++;
 861
 862             if (s == end) {
 863                 FREE_TOKENS;
 864                 return beg;
 865             }
 866
 867             if (*s == '>') {
 868                 s++;
 869                 if (start_com)
 870                     goto FIND_DASH_DASH;
 871
 872                 /* we are done recognizing all comments, make callbacks */
 873                 report_event(p_state, E_COMMENT,
 874                              beg - 4, s, utf8,
 875                              tokens, num_tokens,
 876                              self);
 877                 FREE_TOKENS;
 878
 879                 return s;
 880             }
 881
 882             s++;
 883             if (s == end) {
 884                 FREE_TOKENS;
 885                 return beg;
 886             }
 887
 888             if (*s == '-') {
 889                 /* two dashes in a row seen */
 890                 s++;
 891                 /* do something */
 892                 if (start_com) {
 893                     PUSH_TOKEN(start_com, s-2);
 894                     start_com = 0;
 895                 }
 896                 else {
 897                     start_com = s;
 898                 }
 899             }
 900         }
 901     }
 902     else if (p_state->no_dash_dash_comment_end) {
 903         token_pos_t token;
 904         token.beg = beg;
 905         /* a lone '>' signals end-of-comment */
 906         while (s < end && *s != '>')
 907             s++;
 908         token.end = s;
 909         if (s < end) {
 910             s++;
 911             report_event(p_state, E_COMMENT, beg-4, s, utf8, &token, 1, self);
 912             return s;
 913         }
 914         else {
 915             return beg;
 916         }
 917     }
 918     else { /* non-strict comment */
 919         token_pos_t token;
 920         token.beg = beg;
 921         /* try to locate /--\s*>/ which signals end-of-comment */
 922     LOCATE_END:
 923         while (s < end && *s != '-')
 924             s++;
 925         token.end = s;
 926         if (s < end) {
 927             s++;
 928             if (*s == '-') {
 929                 s++;
 930                 while (isHSPACE(*s))
 931                     s++;
 932                 if (*s == '>') {
 933                     s++;
 934                     /* yup */
 935                     report_event(p_state, E_COMMENT, beg-4, s, utf8, &token, 1, self);
 936                     return s;
 937                 }
 938             }
 939             if (s < end) {
 940                 s = token.end + 1;
 941                 goto LOCATE_END;
 942             }
 943         }
 944
 945         if (s == end)
 946             return beg;
 947     }
 948
 949     return 0;
 950 }
 951
 952
 953 #ifdef MARKED_SECTION
 954
 955 static void
 956 marked_section_update(PSTATE* p_state)
 957 {
 958     dTHX;
 959     /* we look at p_state->ms_stack to determine p_state->ms */
 960     AV* ms_stack = p_state->ms_stack;
 961     p_state->ms = MS_NONE;
 962
 963     if (ms_stack) {
 964         int stack_len = av_len(ms_stack);
 965         int stack_idx;
 966         for (stack_idx = 0; stack_idx <= stack_len; stack_idx++) {
 967             SV** svp = av_fetch(ms_stack, stack_idx, 0);
 968             if (svp) {
 969                 AV* tokens = (AV*)SvRV(*svp);
 970                 int tokens_len = av_len(tokens);
 971                 int i;
 972                 assert(SvTYPE(tokens) == SVt_PVAV);
 973                 for (i = 0; i <= tokens_len; i++) {
 974                     SV** svp = av_fetch(tokens, i, 0);
 975                     if (svp) {
 976                         STRLEN len;
 977                         char *token_str = SvPV(*svp, len);
 978                         enum marked_section_t token;
 979                         if (strEQ(token_str, "include"))
 980                             token = MS_INCLUDE;
 981                         else if (strEQ(token_str, "rcdata"))
 982                             token = MS_RCDATA;
 983                         else if (strEQ(token_str, "cdata"))
 984                             token = MS_CDATA;
 985                         else if (strEQ(token_str, "ignore"))
 986                             token = MS_IGNORE;
 987                         else
 988                             token = MS_NONE;
 989                         if (p_state->ms < token)
 990                             p_state->ms = token;
 991                     }
 992                 }
 993             }
 994         }
 995     }
 996     /* printf("MS %d\n", p_state->ms); */
 997     p_state->is_cdata = (p_state->ms == MS_CDATA);
 998     return;
 999 }
1000
1001
1002 static char*
1003 parse_marked_section(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
1004 {
1005     dTHX;
1006     char *s;
1007     AV* tokens = 0;
1008
1009     if (!p_state->marked_sections)
1010         return 0;
1011
1012     assert(beg[0] == '<');
1013     assert(beg[1] == '!');
1014     assert(beg[2] == '[');
1015     s = beg + 3;
1016
1017 FIND_NAMES:
1018     while (isHSPACE(*s))
1019         s++;
1020     while (isHNAME_FIRST(*s)) {
1021         char *name_start = s;
1022         char *name_end;
1023         SV *name;
1024         s++;
1025         while (isHNAME_CHAR(*s))
1026             s++;
1027         name_end = s;
1028         while (isHSPACE(*s))
1029             s++;
1030         if (s == end)
1031             goto PREMATURE;
1032
1033         if (!tokens)
1034             tokens = newAV();
1035         name = newSVpvn(name_start, name_end - name_start);
1036         if (utf8)
1037             SvUTF8_on(name);
1038         av_push(tokens, sv_lower(aTHX_ name));
1039     }
1040     if (*s == '-') {
1041         s++;
1042         if (*s == '-') {
1043             /* comment */
1044             s++;
1045             while (1) {
1046                 while (s < end && *s != '-')
1047                     s++;
1048                 if (s == end)
1049                     goto PREMATURE;
1050
1051                 s++;  /* skip first '-' */
1052                 if (*s == '-') {
1053                     s++;
1054                     /* comment finished */
1055                     goto FIND_NAMES;
1056                 }
1057             }
1058         }
1059         else
1060             goto FAIL;
1061
1062     }
1063     if (*s == '[') {
1064         s++;
1065         /* yup */
1066
1067         if (!tokens) {
1068             tokens = newAV();
1069             av_push(tokens, newSVpvn("include", 7));
1070         }
1071
1072         if (!p_state->ms_stack)
1073             p_state->ms_stack = newAV();
1074         av_push(p_state->ms_stack, newRV_noinc((SV*)tokens));
1075         marked_section_update(p_state);
1076         report_event(p_state, E_NONE, beg, s, utf8, 0, 0, self);
1077         return s;
1078     }
1079
1080 FAIL:
1081     SvREFCNT_dec(tokens);
1082     return 0; /* not yet implemented */
1083
1084 PREMATURE:
1085     SvREFCNT_dec(tokens);
1086     return beg;
1087 }
1088 #endif
1089
1090
1091 static char*
1092 parse_decl(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
1093 {
1094     char *s = beg + 2;
1095
1096     if (*s == '-') {
1097         /* comment? */
1098
1099         char *tmp;
1100         s++;
1101         if (s == end)
1102             return beg;
1103
1104         if (*s != '-')
1105             goto DECL_FAIL;  /* nope, illegal */
1106
1107         /* yes, two dashes seen */
1108         s++;
1109
1110         tmp = parse_comment(p_state, s, end, utf8, self);
1111         return (tmp == s) ? beg : tmp;
1112     }
1113
1114 #ifdef MARKED_SECTION
1115     if (*s == '[') {
1116         /* marked section */
1117         char *tmp;
1118         tmp = parse_marked_section(p_state, beg, end, utf8, self);
1119         if (!tmp)
1120             goto DECL_FAIL;
1121         return tmp;
1122     }
1123 #endif
1124
1125     if (*s == '>') {
1126         /* make <!> into empty comment <SGML Handbook 36:32> */
1127         token_pos_t token;
1128         token.beg = s;
1129         token.end = s;
1130         s++;
1131         report_event(p_state, E_COMMENT, beg, s, utf8, &token, 1, self);
1132         return s;
1133     }
1134
1135     if (isALPHA(*s)) {
1136         dTOKENS(8);
1137         char *decl_id = s;
1138         STRLEN decl_id_len;
1139
1140         s++;
1141         /* declaration */
1142         while (s < end && isHNAME_CHAR(*s))
1143             s++;
1144         decl_id_len = s - decl_id;
1145         if (s == end)
1146             goto PREMATURE;
1147
1148         /* just hardcode a few names as the recognized declarations */
1149         if (!((decl_id_len == 7 &&
1150                strnEQx(decl_id, "DOCTYPE", 7, !CASE_SENSITIVE(p_state))) ||
1151               (decl_id_len == 6 &&
1152                strnEQx(decl_id, "ENTITY",  6, !CASE_SENSITIVE(p_state)))
1153             )
1154             )
1155         {
1156             goto FAIL;
1157         }
1158
1159         /* first word available */
1160         PUSH_TOKEN(decl_id, s);
1161
1162         while (1) {
1163             while (s < end && isHSPACE(*s))
1164                 s++;
1165
1166             if (s == end)
1167                 goto PREMATURE;
1168
1169             if (*s == '"' || *s == '\'') {
1170                 char *str_beg = s;
1171                 s++;
1172                 while (s < end && *s != *str_beg)
1173                     s++;
1174                 if (s == end)
1175                     goto PREMATURE;
1176                 s++;
1177                 PUSH_TOKEN(str_beg, s);
1178             }
1179             else if (*s == '-') {
1180                 /* comment */
1181                 char *com_beg = s;
1182                 s++;
1183                 if (s == end)
1184                     goto PREMATURE;
1185                 if (*s != '-')
1186                     goto FAIL;
1187                 s++;
1188
1189                 while (1) {
1190                     while (s < end && *s != '-')
1191                         s++;
1192                     if (s == end)
1193                         goto PREMATURE;
1194                     s++;
1195                     if (s == end)
1196                         goto PREMATURE;
1197                     if (*s == '-') {
1198                         s++;
1199                         PUSH_TOKEN(com_beg, s);
1200                         break;
1201                     }
1202                 }
1203             }
1204             else if (*s != '>') {
1205                 /* plain word */
1206                 char *word_beg = s;
1207                 s++;
1208                 while (s < end && isHNOT_SPACE_GT(*s))
1209                     s++;
1210                 if (s == end)
1211                     goto PREMATURE;
1212                 PUSH_TOKEN(word_beg, s);
1213             }
1214             else {
1215                 break;
1216             }
1217         }
1218
1219         if (s == end)
1220             goto PREMATURE;
1221         if (*s == '>') {
1222             s++;
1223             report_event(p_state, E_DECLARATION, beg, s, utf8, tokens, num_tokens, self);
1224             FREE_TOKENS;
1225             return s;
1226         }
1227
1228     FAIL:
1229         FREE_TOKENS;
1230         goto DECL_FAIL;
1231
1232     PREMATURE:
1233         FREE_TOKENS;
1234         return beg;
1235
1236     }
1237
1238 DECL_FAIL:
1239     if (p_state->strict_comment)
1240         return 0;
1241
1242     /* consider everything up to the first '>' a comment */
1243     while (s < end && *s != '>')
1244         s++;
1245     if (s < end) {
1246         token_pos_t token;
1247         token.beg = beg + 2;
1248         token.end = s;
1249         s++;
1250         report_event(p_state, E_COMMENT, beg, s, utf8, &token, 1, self);
1251         return s;
1252     }
1253     else {
1254         return beg;
1255     }
1256 }
1257
1258
1259 static char*
1260 parse_start(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
1261 {
1262     char *s = beg;
1263     int empty_tag = 0;
1264     dTOKENS(16);
1265
1266     hctype_t tag_name_first, tag_name_char;
1267     hctype_t attr_name_first, attr_name_char;
1268
1269     if (STRICT_NAMES(p_state)) {
1270         tag_name_first = attr_name_first = HCTYPE_NAME_FIRST;
1271         tag_name_char  = attr_name_char  = HCTYPE_NAME_CHAR;
1272     }
1273     else {
1274         tag_name_first = tag_name_char = HCTYPE_NOT_SPACE_GT;
1275         attr_name_first = HCTYPE_NOT_SPACE_GT;
1276         attr_name_char  = HCTYPE_NOT_SPACE_EQ_GT;
1277     }
1278
1279     s += 2;
1280
1281     while (s < end && isHCTYPE(*s, tag_name_char)) {
1282         if (*s == '/' && ALLOW_EMPTY_TAG(p_state)) {
1283             if ((s + 1) == end)
1284                 goto PREMATURE;
1285             if (*(s + 1) == '>')
1286                 break;
1287         }
1288         s++;
1289     }
1290     PUSH_TOKEN(beg+1, s);  /* tagname */
1291
1292     while (isHSPACE(*s))
1293         s++;
1294     if (s == end)
1295         goto PREMATURE;
1296
1297     while (isHCTYPE(*s, attr_name_first)) {
1298         /* attribute */
1299         char *attr_name_beg = s;
1300         char *attr_name_end;
1301         if (*s == '/' && ALLOW_EMPTY_TAG(p_state)) {
1302             if ((s + 1) == end)
1303                 goto PREMATURE;
1304             if (*(s + 1) == '>')
1305                 break;
1306         }
1307         s++;
1308         while (s < end && isHCTYPE(*s, attr_name_char)) {
1309             if (*s == '/' && ALLOW_EMPTY_TAG(p_state)) {
1310                 if ((s + 1) == end)
1311                     goto PREMATURE;
1312                 if (*(s + 1) == '>')
1313                     break;
1314             }
1315             s++;
1316         }
1317         if (s == end)
1318             goto PREMATURE;
1319
1320         attr_name_end = s;
1321         PUSH_TOKEN(attr_name_beg, attr_name_end); /* attr name */
1322
1323         while (isHSPACE(*s))
1324             s++;
1325         if (s == end)
1326             goto PREMATURE;
1327
1328         if (*s == '=') {
1329             /* with a value */
1330             s++;
1331             while (isHSPACE(*s))
1332                 s++;
1333             if (s == end)
1334                 goto PREMATURE;
1335             if (*s == '>') {
1336                 /* parse it similar to ="" */
1337                 PUSH_TOKEN(s, s);
1338                 break;
1339             }
1340             if (*s == '"' || *s == '\'') {
1341                 char *str_beg = s;
1342                 s++;
1343                 while (s < end && *s != *str_beg)
1344                     s++;
1345                 if (s == end)
1346                     goto PREMATURE;
1347                 s++;
1348                 PUSH_TOKEN(str_beg, s);
1349             }
1350             else {
1351                 char *word_start = s;
1352                 while (s < end && isHNOT_SPACE_GT(*s)) {
1353                     if (*s == '/' && ALLOW_EMPTY_TAG(p_state)) {
1354                         if ((s + 1) == end)
1355                             goto PREMATURE;
1356                         if (*(s + 1) == '>')
1357                             break;
1358                     }
1359                     s++;
1360                 }
1361                 if (s == end)
1362                     goto PREMATURE;
1363                 PUSH_TOKEN(word_start, s);
1364             }
1365             while (isHSPACE(*s))
1366                 s++;
1367             if (s == end)
1368                 goto PREMATURE;
1369         }
1370         else {
1371             PUSH_TOKEN(0, 0); /* boolean attr value */
1372         }
1373     }
1374
1375     if (ALLOW_EMPTY_TAG(p_state) && *s == '/') {
1376         s++;
1377         if (s == end)
1378             goto PREMATURE;
1379         empty_tag = 1;
1380     }
1381
1382     if (*s == '>') {
1383         s++;
1384         /* done */
1385         report_event(p_state, E_START, beg, s, utf8, tokens, num_tokens, self);
1386         if (empty_tag) {
1387             report_event(p_state, E_END, s, s, utf8, tokens, 1, self);
1388         }
1389         else if (!p_state->xml_mode) {
1390             /* find out if this start tag should put us into literal_mode
1391              */
1392             int i;
1393             int tag_len = tokens[0].end - tokens[0].beg;
1394
1395             for (i = 0; literal_mode_elem[i].len; i++) {
1396                 if (tag_len == literal_mode_elem[i].len) {
1397                     /* try to match it */
1398                     char *s = beg + 1;
1399                     char *t = literal_mode_elem[i].str;
1400                     int len = tag_len;
1401                     while (len) {
1402                         if (toLOWER(*s) != *t)
1403                             break;
1404                         s++;
1405                         t++;
1406                         if (!--len) {
1407                             /* found it */
1408                             p_state->literal_mode = literal_mode_elem[i].str;
1409                             p_state->is_cdata = literal_mode_elem[i].is_cdata;
1410                             /* printf("Found %s\n", p_state->literal_mode); */
1411                             goto END_OF_LITERAL_SEARCH;
1412                         }
1413                     }
1414                 }
1415             }
1416         END_OF_LITERAL_SEARCH:
1417             ;
1418         }
1419
1420         FREE_TOKENS;
1421         return s;
1422     }
1423
1424     FREE_TOKENS;
1425     return 0;
1426
1427 PREMATURE:
1428     FREE_TOKENS;
1429     return beg;
1430 }
1431
1432
1433 static char*
1434 parse_end(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
1435 {
1436     char *s = beg+2;
1437     hctype_t name_first, name_char;
1438
1439     if (STRICT_NAMES(p_state)) {
1440         name_first = HCTYPE_NAME_FIRST;
1441         name_char  = HCTYPE_NAME_CHAR;
1442     }
1443     else {
1444         name_first = name_char = HCTYPE_NOT_SPACE_GT;
1445     }
1446
1447     if (isHCTYPE(*s, name_first)) {
1448         token_pos_t tagname;
1449         tagname.beg = s;
1450         s++;
1451         while (s < end && isHCTYPE(*s, name_char))
1452             s++;
1453         tagname.end = s;
1454
1455         if (p_state->strict_end) {
1456             while (isHSPACE(*s))
1457                 s++;
1458         }
1459         else {
1460             s = skip_until_gt(s, end);
1461         }
1462         if (s < end) {
1463             if (*s == '>') {
1464                 s++;
1465                 /* a complete end tag has been recognized */
1466                 report_event(p_state, E_END, beg, s, utf8, &tagname, 1, self);
1467                 return s;
1468             }
1469         }
1470         else {
1471             return beg;
1472         }
1473     }
1474     else if (!p_state->strict_comment) {
1475         s = skip_until_gt(s, end);
1476         if (s < end) {
1477             token_pos_t token;
1478             token.beg = beg + 2;
1479             token.end = s;
1480             s++;
1481             report_event(p_state, E_COMMENT, beg, s, utf8, &token, 1, self);
1482             return s;
1483         }
1484         else {
1485             return beg;
1486         }
1487     }
1488     return 0;
1489 }
1490
1491
1492 static char*
1493 parse_process(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
1494 {
1495     char *s = beg + 2;  /* skip '<?' */
1496     /* processing instruction */
1497     token_pos_t token_pos;
1498     token_pos.beg = s;
1499
1500     while (s < end) {
1501         if (*s == '>') {
1502             token_pos.end = s;
1503             s++;
1504
1505             if (p_state->xml_mode || p_state->xml_pic) {
1506                 /* XML processing instructions are ended by "?>" */
1507                 if (s - beg < 4 || s[-2] != '?')
1508                     continue;
1509                 token_pos.end = s - 2;
1510             }
1511
1512             /* a complete processing instruction seen */
1513             report_event(p_state, E_PROCESS, beg, s, utf8,
1514                          &token_pos, 1, self);
1515             return s;
1516         }
1517         s++;
1518     }
1519     return beg;  /* could not fix end */
1520 }
1521
1522
1523 #ifdef USE_PFUNC
1524 static char*
1525 parse_null(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
1526 {
1527     return 0;
1528 }
1529
1530
1531
1532 #include "pfunc.h"                   /* declares the parsefunc[] */
1533 #endif /* USE_PFUNC */
1534
1535 static char*
1536 parse_buf(pTHX_ PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
1537 {
1538     char *s = beg;
1539     char *t = beg;
1540     char *new_pos;
1541
1542     while (!p_state->eof) {
1543         /*
1544          * At the start of this loop we will always be ready for eating text
1545          * or a new tag.  We will never be inside some tag.  The 't' points
1546          * to where we started and the 's' is advanced as we go.
1547          */
1548
1549         while (p_state->literal_mode) {
1550             char *l = p_state->literal_mode;
1551             bool skip_quoted_end = (strEQ(l, "script") || strEQ(l, "style"));
1552             char inside_quote = 0;
1553             bool escape_next = 0;
1554             char *end_text;
1555
1556             while (s < end) {
1557                 if (*s == '<' && !inside_quote)
1558                     break;
1559                 if (skip_quoted_end) {
1560                     if (escape_next) {
1561                         escape_next = 0;
1562                     }
1563                     else {
1564                         if (*s == '\\')
1565                             escape_next = 1;
1566                         else if (inside_quote && *s == inside_quote)
1567                             inside_quote = 0;
1568                         else if (*s == '\r' || *s == '\n')
1569                             inside_quote = 0;
1570                         else if (!inside_quote && (*s == '"' || *s == '\''))
1571                             inside_quote = *s;
1572                     }
1573                 }
1574                 s++;
1575             }
1576
1577             if (s == end) {
1578                 s = t;
1579                 goto DONE;
1580             }
1581
1582             end_text = s;
1583             s++;
1584
1585             /* here we rely on '\0' termination of perl svpv buffers */
1586             if (*s == '/') {
1587                 s++;
1588                 while (*l && toLOWER(*s) == *l) {
1589                     s++;
1590                     l++;
1591                 }
1592
1593                 if (!*l && (strNE(p_state->literal_mode, "plaintext") || p_state->closing_plaintext)) {
1594                     /* matched it all */
1595                     token_pos_t end_token;
1596                     end_token.beg = end_text + 2;
1597                     end_token.end = s;
1598
1599                     while (isHSPACE(*s))
1600                         s++;
1601                     if (*s == '>') {
1602                         s++;
1603                         if (t != end_text)
1604                             report_event(p_state, E_TEXT, t, end_text, utf8,
1605                                          0, 0, self);
1606                         report_event(p_state, E_END,  end_text, s, utf8,
1607                                      &end_token, 1, self);
1608                         p_state->literal_mode = 0;
1609                         p_state->is_cdata = 0;
1610                         t = s;
1611                     }
1612                 }
1613             }
1614         }
1615
1616 #ifdef MARKED_SECTION
1617         while (p_state->ms == MS_CDATA || p_state->ms == MS_RCDATA) {
1618             while (s < end && *s != ']')
1619                 s++;
1620             if (*s == ']') {
1621                 char *end_text = s;
1622                 s++;
1623                 if (*s == ']' && *(s + 1) == '>') {
1624                     s += 2;
1625                     /* marked section end */
1626                     if (t != end_text)
1627                         report_event(p_state, E_TEXT, t, end_text, utf8,
1628                                      0, 0, self);
1629                     report_event(p_state, E_NONE, end_text, s, utf8, 0, 0, self);
1630                     t = s;
1631                     SvREFCNT_dec(av_pop(p_state->ms_stack));
1632                     marked_section_update(p_state);
1633                     continue;
1634                 }
1635             }
1636             if (s == end) {
1637                 s = t;
1638                 goto DONE;
1639             }
1640         }
1641 #endif
1642
1643         /* first we try to match as much text as possible */
1644         while (s < end && *s != '<') {
1645 #ifdef MARKED_SECTION
1646             if (p_state->ms && *s == ']') {
1647                 char *end_text = s;
1648                 s++;
1649                 if (*s == ']') {
1650                     s++;
1651                     if (*s == '>') {
1652                         s++;
1653                         report_event(p_state, E_TEXT, t, end_text, utf8,
1654                                      0, 0, self);
1655                         report_event(p_state, E_NONE, end_text, s, utf8,
1656                                      0, 0, self);
1657                         t = s;
1658                         SvREFCNT_dec(av_pop(p_state->ms_stack));
1659                         marked_section_update(p_state);
1660                         continue;
1661                     }
1662                 }
1663             }
1664 #endif
1665             s++;
1666         }
1667         if (s != t) {
1668             if (*s == '<') {
1669                 report_event(p_state, E_TEXT, t, s, utf8, 0, 0, self);
1670                 t = s;
1671             }
1672             else {
1673                 s--;
1674                 if (isHSPACE(*s)) {
1675                     /* wait with white space at end */
1676                     while (s >= t && isHSPACE(*s))
1677                         s--;
1678                 }
1679                 else {
1680                     /* might be a chopped up entities/words */
1681                     while (s >= t && !isHSPACE(*s))
1682                         s--;
1683                     while (s >= t && isHSPACE(*s))
1684                         s--;
1685                 }
1686                 s++;
1687                 if (s != t)
1688                     report_event(p_state, E_TEXT, t, s, utf8, 0, 0, self);
1689                 break;
1690             }
1691         }
1692
1693         if (end - s < 3)
1694             break;
1695
1696         /* next char is known to be '<' and pointed to by 't' as well as 's' */
1697         s++;
1698
1699 #ifdef USE_PFUNC
1700         new_pos = parsefunc[(unsigned char)*s](p_state, t, end, utf8, self);
1701 #else
1702         if (isHNAME_FIRST(*s))
1703             new_pos = parse_start(p_state, t, end, utf8, self);
1704         else if (*s == '/')
1705             new_pos = parse_end(p_state, t, end, utf8, self);
1706         else if (*s == '!')
1707             new_pos = parse_decl(p_state, t, end, utf8, self);
1708         else if (*s == '?')
1709             new_pos = parse_process(p_state, t, end, utf8, self);
1710         else
1711             new_pos = 0;
1712 #endif /* USE_PFUNC */
1713
1714         if (new_pos) {
1715             if (new_pos == t) {
1716                 /* no progress, need more data to know what it is */
1717                 s = t;
1718                 break;
1719             }
1720             t = s = new_pos;
1721         }
1722
1723         /* if we get out here then this was not a conforming tag, so
1724          * treat it is plain text at the top of the loop again (we
1725          * have already skipped past the "<").
1726          */
1727     }
1728
1729 DONE:
1730     return s;
1731
1732 }
1733
1734 EXTERN void
1735 parse(pTHX_
1736       PSTATE* p_state,
1737       SV* chunk,
1738       SV* self)
1739 {
1740     char *s, *beg, *end;
1741     U32 utf8 = 0;
1742     STRLEN len;
1743
1744     if (!p_state->start_document) {
1745         char dummy[1];
1746         report_event(p_state, E_START_DOCUMENT, dummy, dummy, 0, 0, 0, self);
1747         p_state->start_document = 1;
1748     }
1749
1750     if (!chunk) {
1751         /* eof */
1752         char empty[1];
1753         if (p_state->buf && SvOK(p_state->buf)) {
1754             /* flush it */
1755             s = SvPV(p_state->buf, len);
1756             end = s + len;
1757             utf8 = SvUTF8(p_state->buf);
1758             assert(len);
1759
1760             while (s < end) {
1761                 if (p_state->literal_mode) {
1762                     if (strEQ(p_state->literal_mode, "plaintext") ||
1763                         strEQ(p_state->literal_mode, "xmp") ||
1764                         strEQ(p_state->literal_mode, "textarea"))
1765                     {
1766                         /* rest is considered text */
1767                         break;
1768                     }
1769                     if (strEQ(p_state->literal_mode, "script") ||
1770                         strEQ(p_state->literal_mode, "style"))
1771                     {
1772                         /* effectively make it an empty element */
1773                         token_pos_t t;
1774                         char dummy;
1775                         t.beg = p_state->literal_mode;
1776                         t.end = p_state->literal_mode + strlen(p_state->literal_mode);
1777                         report_event(p_state, E_END, &dummy, &dummy, 0, &t, 1, self);
1778                     }
1779                     else {
1780                         p_state->pending_end_tag = p_state->literal_mode;
1781                     }
1782                     p_state->literal_mode = 0;
1783                     s = parse_buf(aTHX_ p_state, s, end, utf8, self);
1784                     continue;
1785                 }
1786
1787                 if (!p_state->strict_comment && !p_state->no_dash_dash_comment_end && *s == '<') {
1788                     p_state->no_dash_dash_comment_end = 1;
1789                     s = parse_buf(aTHX_ p_state, s, end, utf8, self);
1790                     continue;
1791                 }
1792
1793                 if (!p_state->strict_comment && *s == '<') {
1794                     char *s1 = s + 1;
1795                     if (s1 == end || isHNAME_FIRST(*s1) || *s1 == '/' || *s1 == '!' || *s1 == '?') {
1796                         /* some kind of unterminated markup.  Report rest as as comment */
1797                         token_pos_t token;
1798                         token.beg = s + 1;
1799                         token.end = end;
1800                         report_event(p_state, E_COMMENT, s, end, utf8, &token, 1, self);
1801                         s = end;
1802                     }
1803                 }
1804
1805                 break;
1806             }
1807
1808             if (s < end) {
1809                 /* report rest as text */
1810                 report_event(p_state, E_TEXT, s, end, utf8, 0, 0, self);
1811             }
1812
1813             SvREFCNT_dec(p_state->buf);
1814             p_state->buf = 0;
1815         }
1816         if (p_state->pend_text && SvOK(p_state->pend_text))
1817             flush_pending_text(p_state, self);
1818
1819         if (p_state->ignoring_element) {
1820             /* document not balanced */
1821             SvREFCNT_dec(p_state->ignoring_element);
1822             p_state->ignoring_element = 0;
1823         }
1824         report_event(p_state, E_END_DOCUMENT, empty, empty, 0, 0, 0, self);
1825
1826         /* reset state */
1827         p_state->offset = 0;
1828         if (p_state->line)
1829             p_state->line = 1;
1830         p_state->column = 0;
1831         p_state->start_document = 0;
1832         p_state->literal_mode = 0;
1833         p_state->is_cdata = 0;
1834         return;
1835     }
1836
1837 #ifdef UNICODE_HTML_PARSER
1838     if (p_state->utf8_mode)
1839         sv_utf8_downgrade(chunk, 0);
1840 #endif
1841
1842     if (p_state->buf && SvOK(p_state->buf)) {
1843         sv_catsv(p_state->buf, chunk);
1844         beg = SvPV(p_state->buf, len);
1845         utf8 = SvUTF8(p_state->buf);
1846     }
1847     else {
1848         beg = SvPV(chunk, len);
1849         utf8 = SvUTF8(chunk);
1850         if (p_state->offset == 0 && DOWARN) {
1851             /* Print warnings if we find unexpected Unicode BOM forms */
1852 #ifdef UNICODE_HTML_PARSER
1853             if (p_state->argspec_entity_decode &&
1854                 !p_state->utf8_mode && (
1855                  (!utf8 && len >= 3 && strnEQ(beg, "\xEF\xBB\xBF", 3)) ||
1856                  (utf8 && len >= 6 && strnEQ(beg, "\xC3\xAF\xC2\xBB\xC2\xBF", 6)) ||
1857                  (!utf8 && probably_utf8_chunk(aTHX_ beg, len))
1858                 )
1859                )
1860             {
1861                 warn("Parsing of undecoded UTF-8 will give garbage when decoding entities");
1862             }
1863             if (utf8 && len >= 2 && strnEQ(beg, "\xFF\xFE", 2)) {
1864                 warn("Parsing string decoded with wrong endianess");
1865             }
1866 #endif
1867             if (!utf8 && len >= 4 &&
1868                 (strnEQ(beg, "\x00\x00\xFE\xFF", 4) ||
1869                  strnEQ(beg, "\xFE\xFF\x00\x00", 4))
1870                 )
1871             {
1872                 warn("Parsing of undecoded UTF-32");
1873             }
1874             else if (!utf8 && len >= 2 &&
1875                      (strnEQ(beg, "\xFE\xFF", 2) || strnEQ(beg, "\xFF\xFE", 2))
1876                 )
1877             {
1878                 warn("Parsing of undecoded UTF-16");
1879             }
1880         }
1881     }
1882
1883     if (!len)
1884         return; /* nothing to do */
1885
1886     end = beg + len;
1887     s = parse_buf(aTHX_ p_state, beg, end, utf8, self);
1888
1889     if (s == end || p_state->eof) {
1890         if (p_state->buf) {
1891             SvOK_off(p_state->buf);
1892         }
1893     }
1894     else {
1895         /* need to keep rest in buffer */
1896         if (p_state->buf) {
1897             /* chop off some chars at the beginning */
1898             if (SvOK(p_state->buf)) {
1899                 sv_chop(p_state->buf, s);
1900             }
1901             else {
1902                 sv_setpvn(p_state->buf, s, end - s);
1903                 if (utf8)
1904                     SvUTF8_on(p_state->buf);
1905                 else
1906                     SvUTF8_off(p_state->buf);
1907             }
1908         }
1909         else {
1910             p_state->buf = newSVpv(s, end - s);
1911             if (utf8)
1912                 SvUTF8_on(p_state->buf);
1913         }
1914     }
1915     return;
1916 }