duk_lexer.c File Reference

#include "duk_internal.h"

Go to the source code of this file.

Macros
#define	DUK__MAX_RE_DECESC_DIGITS   9
#define	DUK__MAX_RE_QUANT_DIGITS   9 /* Does not allow e.g. 2**31-1, but one more would allow overflows of u32. */
#define	DUK__ISDIGIT(x)   ((x) >= DUK_ASC_0 && (x) <= DUK_ASC_9)
#define	DUK__ISHEXDIGIT(x)   duk__is_hex_digit((x))
#define	DUK__ISOCTDIGIT(x)   ((x) >= DUK_ASC_0 && (x) <= DUK_ASC_7)
#define	DUK__ISDIGIT03(x)   ((x) >= DUK_ASC_0 && (x) <= DUK_ASC_3)
#define	DUK__ISDIGIT47(x)   ((x) >= DUK_ASC_4 && (x) <= DUK_ASC_7)
#define	DUK__LOOKUP(lex_ctx, index)   ((lex_ctx)->window[(index)].codepoint)
#define	DUK__ADVANCECHARS(lex_ctx, count)   duk__advance_bytes((lex_ctx), (count) * sizeof(duk_lexer_codepoint))
#define	DUK__ADVANCEBYTES(lex_ctx, count)   duk__advance_bytes((lex_ctx), (count))
#define	DUK__INITBUFFER(lex_ctx)   duk__initbuffer((lex_ctx))
#define	DUK__APPENDBUFFER(lex_ctx, x)   duk__appendbuffer((lex_ctx), (duk_codepoint_t) (x))
#define	DUK__L0()   DUK__LOOKUP(lex_ctx, 0)
#define	DUK__L1()   DUK__LOOKUP(lex_ctx, 1)
#define	DUK__L2()   DUK__LOOKUP(lex_ctx, 2)
#define	DUK__L3()   DUK__LOOKUP(lex_ctx, 3)
#define	DUK__L4()   DUK__LOOKUP(lex_ctx, 4)
#define	DUK__L5()   DUK__LOOKUP(lex_ctx, 5)
#define	DUK__ADVTOK(advbytes, tok)   ((((advbytes) * sizeof(duk_lexer_codepoint)) << 8) + (tok))

Functions
DUK_LOCAL void	duk__fill_lexer_buffer (duk_lexer_ctx *lex_ctx, duk_small_uint_t start_offset_bytes)
DUK_LOCAL void	duk__advance_bytes (duk_lexer_ctx *lex_ctx, duk_small_uint_t count_bytes)
DUK_LOCAL void	duk__init_lexer_window (duk_lexer_ctx *lex_ctx)
DUK_LOCAL void	duk__initbuffer (duk_lexer_ctx *lex_ctx)
DUK_LOCAL void	duk__appendbuffer (duk_lexer_ctx *lex_ctx, duk_codepoint_t x)
DUK_LOCAL void	duk__internbuffer (duk_lexer_ctx *lex_ctx, duk_idx_t valstack_idx)
DUK_INTERNAL void	duk_lexer_initctx (duk_lexer_ctx *lex_ctx)
DUK_INTERNAL void	duk_lexer_setpoint (duk_lexer_ctx *lex_ctx, duk_lexer_point *pt)
DUK_LOCAL duk_codepoint_t	duk__hexval (duk_lexer_ctx *lex_ctx, duk_codepoint_t x)
DUK_LOCAL duk_bool_t	duk__is_hex_digit (duk_codepoint_t x)
DUK_LOCAL duk_codepoint_t	duk__decode_hexesc_from_window (duk_lexer_ctx *lex_ctx, duk_small_int_t lookup_offset)
DUK_LOCAL duk_codepoint_t	duk__decode_uniesc_from_window (duk_lexer_ctx *lex_ctx, duk_small_int_t lookup_offset)
DUK_INTERNAL void	duk_lexer_parse_js_input_element (duk_lexer_ctx *lex_ctx, duk_token *out_token, duk_bool_t strict_mode, duk_bool_t regexp_mode)
DUK_INTERNAL void	duk_lexer_parse_re_token (duk_lexer_ctx *lex_ctx, duk_re_token *out_token)
DUK_LOCAL void	duk__emit_u16_direct_ranges (duk_lexer_ctx *lex_ctx, duk_re_range_callback gen_range, void *userdata, const duk_uint16_t *ranges, duk_small_int_t num)
DUK_INTERNAL void	duk_lexer_parse_re_ranges (duk_lexer_ctx *lex_ctx, duk_re_range_callback gen_range, void *userdata)

Macro Definition Documentation

DUK__ADVANCEBYTES

#define DUK__ADVANCEBYTES	(		lex_ctx,
			count )   duk__advance_bytes((lex_ctx), (count))

Definition at line 92 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element(), and duk_lexer_parse_re_token().

DUK__ADVANCECHARS

#define DUK__ADVANCECHARS	(		lex_ctx,
			count )   duk__advance_bytes((lex_ctx), (count) * sizeof(duk_lexer_codepoint))

Definition at line 91 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element(), duk_lexer_parse_re_ranges(), and duk_lexer_parse_re_token().

DUK__ADVTOK

#define DUK__ADVTOK	(		advbytes,
			tok )   ((((advbytes) * sizeof(duk_lexer_codepoint)) << 8) + (tok))

Definition at line 105 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element(), and duk_lexer_parse_re_token().

DUK__APPENDBUFFER

#define DUK__APPENDBUFFER	(		lex_ctx,
			x )   duk__appendbuffer((lex_ctx), (duk_codepoint_t) (x))

Definition at line 94 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element().

DUK__INITBUFFER

#define DUK__INITBUFFER

(

lex_ctx

)

duk__initbuffer((lex_ctx))

Definition at line 93 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element().

DUK__ISDIGIT

#define DUK__ISDIGIT

(

x

)

((x) >= DUK_ASC_0 && (x) <= DUK_ASC_9)

Definition at line 83 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element(), duk_lexer_parse_re_ranges(), and duk_lexer_parse_re_token().

DUK__ISDIGIT03

#define DUK__ISDIGIT03

(

x

)

((x) >= DUK_ASC_0 && (x) <= DUK_ASC_3)

Definition at line 86 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element().

DUK__ISDIGIT47

#define DUK__ISDIGIT47

(

x

)

((x) >= DUK_ASC_4 && (x) <= DUK_ASC_7)

Definition at line 87 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element().

DUK__ISHEXDIGIT

#define DUK__ISHEXDIGIT

(

x

)

duk__is_hex_digit((x))

Definition at line 84 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element().

DUK__ISOCTDIGIT

#define DUK__ISOCTDIGIT

(

x

)

((x) >= DUK_ASC_0 && (x) <= DUK_ASC_7)

Definition at line 85 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element().

DUK__L0

#define DUK__L0

(

)

DUK__LOOKUP(lex_ctx, 0)

Definition at line 97 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element(), duk_lexer_parse_re_ranges(), and duk_lexer_parse_re_token().

DUK__L1

#define DUK__L1

(

)

DUK__LOOKUP(lex_ctx, 1)

Definition at line 98 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element(), and duk_lexer_parse_re_token().

DUK__L2

#define DUK__L2

(

)

DUK__LOOKUP(lex_ctx, 2)

Definition at line 99 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element(), and duk_lexer_parse_re_token().

DUK__L3

#define DUK__L3

(

)

DUK__LOOKUP(lex_ctx, 3)

Definition at line 100 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element().

DUK__L4

#define DUK__L4

(

)

DUK__LOOKUP(lex_ctx, 4)

Definition at line 101 of file duktape-1.5.2/src-separate/duk_lexer.c.

DUK__L5

#define DUK__L5

(

)

DUK__LOOKUP(lex_ctx, 5)

Definition at line 102 of file duktape-1.5.2/src-separate/duk_lexer.c.

DUK__LOOKUP

#define DUK__LOOKUP	(		lex_ctx,
			index )   ((lex_ctx)->window[(index)].codepoint)

Definition at line 90 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_js_input_element().

DUK__MAX_RE_DECESC_DIGITS

#define DUK__MAX_RE_DECESC_DIGITS   9

Definition at line 79 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_re_token().

DUK__MAX_RE_QUANT_DIGITS

#define DUK__MAX_RE_QUANT_DIGITS   9 /* Does not allow e.g. 2**31-1, but one more would allow overflows of u32. */

Definition at line 80 of file duktape-1.5.2/src-separate/duk_lexer.c.

Referenced by duk_lexer_parse_re_token().

Function Documentation

duk__advance_bytes()

DUK_LOCAL void duk__advance_bytes	(	duk_lexer_ctx *	lex_ctx,
		duk_small_uint_t	count_bytes )

Definition at line 317 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                        {
    duk_small_uint_t used_bytes, avail_bytes;
 
    DUK_ASSERT_DISABLE(count_bytes >= 0);  /* unsigned */
    DUK_ASSERT(count_bytes <= (duk_small_uint_t) (DUK_LEXER_WINDOW_SIZE * sizeof(duk_lexer_codepoint)));
    DUK_ASSERT(lex_ctx->window >= lex_ctx->buffer);
    DUK_ASSERT(lex_ctx->window < lex_ctx->buffer + DUK_LEXER_BUFFER_SIZE);
    DUK_ASSERT((duk_uint8_t *) lex_ctx->window + count_bytes <= (duk_uint8_t *) lex_ctx->buffer + DUK_LEXER_BUFFER_SIZE * sizeof(duk_lexer_codepoint));
 
    /* Zero 'count' is also allowed to make call sites easier.
     * Arithmetic in bytes generates better code in GCC.
     */
 
    lex_ctx->window = (duk_lexer_codepoint *) (void *) ((duk_uint8_t *) lex_ctx->window + count_bytes);  /* avoid multiply */
    used_bytes = (duk_small_uint_t) ((duk_uint8_t *) lex_ctx->window - (duk_uint8_t *) lex_ctx->buffer);
    avail_bytes = DUK_LEXER_BUFFER_SIZE * sizeof(duk_lexer_codepoint) - used_bytes;
    if (avail_bytes < (duk_small_uint_t) (DUK_LEXER_WINDOW_SIZE * sizeof(duk_lexer_codepoint))) {
        /* Not enough data to provide a full window, so "scroll" window to
         * start of buffer and fill up the rest.
         */
        DUK_MEMMOVE((void *) lex_ctx->buffer,
                    (const void *) lex_ctx->window,
                    (size_t) avail_bytes);
        lex_ctx->window = lex_ctx->buffer;
        duk__fill_lexer_buffer(lex_ctx, avail_bytes);
    }
}

References duk_lexer_ctx::buffer, duk__fill_lexer_buffer(), DUK_ASSERT, DUK_ASSERT_DISABLE, DUK_LEXER_BUFFER_SIZE, DUK_LEXER_WINDOW_SIZE, DUK_MEMMOVE, and duk_lexer_ctx::window.

duk__appendbuffer()

DUK_LOCAL void duk__appendbuffer	(	duk_lexer_ctx *	lex_ctx,
		duk_codepoint_t	x )

Definition at line 529 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                            {
    /*
     *  Since character data is only generated by decoding the source or by
     *  the compiler itself, we rely on the input codepoints being correct
     *  and avoid a check here.
     *
     *  Character data can also come here through decoding of Unicode
     *  escapes ("\udead\ubeef") so all 16-but unsigned values can be
     *  present, even when the source file itself is strict UTF-8.
     */
 
    DUK_ASSERT(x >= 0 && x <= 0x10ffff);
 
    DUK_BW_WRITE_ENSURE_CESU8(lex_ctx->thr, &lex_ctx->bw, (duk_ucodepoint_t) x);
}

References duk_lexer_ctx::bw, DUK_ASSERT, DUK_BW_WRITE_ENSURE_CESU8, and duk_lexer_ctx::thr.

duk__decode_hexesc_from_window()

DUK_LOCAL duk_codepoint_t duk__decode_hexesc_from_window	(	duk_lexer_ctx *	lex_ctx,
		duk_small_int_t	lookup_offset )

Definition at line 622 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                                                {
    /* validation performed by duk__hexval */
    return (duk__hexval(lex_ctx, lex_ctx->window[lookup_offset].codepoint) << 4) |
           (duk__hexval(lex_ctx, lex_ctx->window[lookup_offset + 1].codepoint));
}

References duk_lexer_codepoint::codepoint, duk__hexval(), and duk_lexer_ctx::window.

Referenced by duk_lexer_parse_js_input_element(), duk_lexer_parse_re_ranges(), and duk_lexer_parse_re_token().

duk__decode_uniesc_from_window()

DUK_LOCAL duk_codepoint_t duk__decode_uniesc_from_window	(	duk_lexer_ctx *	lex_ctx,
		duk_small_int_t	lookup_offset )

Definition at line 628 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                                                {
    /* validation performed by duk__hexval */
    return (duk__hexval(lex_ctx, lex_ctx->window[lookup_offset].codepoint) << 12) |
           (duk__hexval(lex_ctx, lex_ctx->window[lookup_offset + 1].codepoint) << 8) |
           (duk__hexval(lex_ctx, lex_ctx->window[lookup_offset + 2].codepoint) << 4) |
           (duk__hexval(lex_ctx, lex_ctx->window[lookup_offset + 3].codepoint));
}

References duk_lexer_codepoint::codepoint, duk__hexval(), and duk_lexer_ctx::window.

Referenced by duk_lexer_parse_js_input_element(), duk_lexer_parse_re_ranges(), and duk_lexer_parse_re_token().

duk__emit_u16_direct_ranges()

DUK_LOCAL void duk__emit_u16_direct_ranges	(	duk_lexer_ctx *	lex_ctx,
		duk_re_range_callback	gen_range,
		void *	userdata,
		const duk_uint16_t *	ranges,
		duk_small_int_t	num )

Definition at line 1868 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                      {
    const duk_uint16_t *ranges_end;
 
    DUK_UNREF(lex_ctx);
 
    ranges_end = ranges + num;
    while (ranges < ranges_end) {
        /* mark range 'direct', bypass canonicalization (see Wiki) */
        gen_range(userdata, (duk_codepoint_t) ranges[0], (duk_codepoint_t) ranges[1], 1);
        ranges += 2;
    }
}

References DUK_UNREF.

Referenced by duk_lexer_parse_re_ranges().

duk__fill_lexer_buffer()

DUK_LOCAL void duk__fill_lexer_buffer	(	duk_lexer_ctx *	lex_ctx,
		duk_small_uint_t	start_offset_bytes )

Definition at line 177 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                                   {
    duk_lexer_codepoint *cp, *cp_end;
    duk_ucodepoint_t x;
    duk_small_uint_t contlen;
    const duk_uint8_t *p, *p_end;
#if defined(DUK_USE_STRICT_UTF8_SOURCE)
    duk_ucodepoint_t mincp;
#endif
    duk_int_t input_line;
 
    /* Use temporaries and update lex_ctx only when finished. */
    input_line = lex_ctx->input_line;
    p = lex_ctx->input + lex_ctx->input_offset;
    p_end = lex_ctx->input + lex_ctx->input_length;
 
    cp = (duk_lexer_codepoint *) (void *) ((duk_uint8_t *) lex_ctx->buffer + start_offset_bytes);
    cp_end = lex_ctx->buffer + DUK_LEXER_BUFFER_SIZE;
 
    for (; cp != cp_end; cp++) {
        cp->offset = (duk_size_t) (p - lex_ctx->input);
        cp->line = input_line;
 
        /* XXX: potential issue with signed pointers, p_end < p. */
        if (DUK_UNLIKELY(p >= p_end)) {
            /* If input_offset were assigned a negative value, it would
             * result in a large positive value.  Most likely it would be
             * larger than input_length and be caught here.  In any case
             * no memory unsafe behavior would happen.
             */
            cp->codepoint = -1;
            continue;
        }
 
        x = (duk_ucodepoint_t) (*p++);
 
        /* Fast path. */
 
        if (DUK_LIKELY(x < 0x80UL)) {
            DUK_ASSERT(x != 0x2028UL && x != 0x2029UL);  /* not LS/PS */
            if (DUK_UNLIKELY(x <= 0x000dUL)) {
                if ((x == 0x000aUL) ||
                    ((x == 0x000dUL) && (p >= p_end || *p != 0x000aUL))) {
                    /* lookup for 0x000a above assumes shortest encoding now */
 
                    /* E5 Section 7.3, treat the following as newlines:
                     *   LF
                     *   CR [not followed by LF]
                     *   LS
                     *   PS
                     *
                     * For CR LF, CR is ignored if it is followed by LF, and the LF will bump
                     * the line number.
                     */
                    input_line++;
                }
            }
 
            cp->codepoint = (duk_codepoint_t) x;
            continue;
        }
 
        /* Slow path. */
 
        if (x < 0xc0UL) {
            /* 10xx xxxx -> invalid */
            goto error_encoding;
        } else if (x < 0xe0UL) {
            /* 110x xxxx   10xx xxxx  */
            contlen = 1;
#if defined(DUK_USE_STRICT_UTF8_SOURCE)
            mincp = 0x80UL;
#endif
            x = x & 0x1fUL;
        } else if (x < 0xf0UL) {
            /* 1110 xxxx   10xx xxxx   10xx xxxx */
            contlen = 2;
#if defined(DUK_USE_STRICT_UTF8_SOURCE)
            mincp = 0x800UL;
#endif
            x = x & 0x0fUL;
        } else if (x < 0xf8UL) {
            /* 1111 0xxx   10xx xxxx   10xx xxxx   10xx xxxx */
            contlen = 3;
#if defined(DUK_USE_STRICT_UTF8_SOURCE)
            mincp = 0x10000UL;
#endif
            x = x & 0x07UL;
        } else {
            /* no point in supporting encodings of 5 or more bytes */
            goto error_encoding;
        }
 
        DUK_ASSERT(p_end >= p);
        if ((duk_size_t) contlen > (duk_size_t) (p_end - p)) {
            goto error_clipped;
        }
 
        while (contlen > 0) {
            duk_small_uint_t y;
            y = *p++;
            if ((y & 0xc0U) != 0x80U) {
                /* check that byte has the form 10xx xxxx */
                goto error_encoding;
            }
            x = x << 6;
            x += y & 0x3fUL;
            contlen--;
        }
 
        /* check final character validity */
 
        if (x > 0x10ffffUL) {
            goto error_encoding;
        }
#if defined(DUK_USE_STRICT_UTF8_SOURCE)
        if (x < mincp || (x >= 0xd800UL && x <= 0xdfffUL) || x == 0xfffeUL) {
            goto error_encoding;
        }
#endif
 
        DUK_ASSERT(x != 0x000aUL && x != 0x000dUL);
        if ((x == 0x2028UL) || (x == 0x2029UL)) {
            input_line++;
        }
 
        cp->codepoint = (duk_codepoint_t) x;
    }
 
    lex_ctx->input_offset = (duk_size_t) (p - lex_ctx->input);
    lex_ctx->input_line = input_line;
    return;
 
 error_clipped:   /* clipped codepoint */
 error_encoding:  /* invalid codepoint encoding or codepoint */
    lex_ctx->input_offset = (duk_size_t) (p - lex_ctx->input);
    lex_ctx->input_line = input_line;
 
    DUK_ERROR_SYNTAX(lex_ctx->thr, "utf-8 decode failed");
}

References duk_lexer_ctx::buffer, duk_lexer_codepoint::codepoint, DUK_ASSERT, DUK_ERROR_SYNTAX, DUK_LEXER_BUFFER_SIZE, DUK_LIKELY, DUK_UNLIKELY, duk_lexer_ctx::input, duk_lexer_ctx::input_length, duk_lexer_ctx::input_line, duk_lexer_ctx::input_offset, duk_lexer_codepoint::line, duk_lexer_codepoint::offset, and duk_lexer_ctx::thr.

Referenced by duk__advance_bytes(), and duk__init_lexer_window().

duk__hexval()

DUK_LOCAL duk_codepoint_t duk__hexval	(	duk_lexer_ctx *	lex_ctx,
		duk_codepoint_t	x )

Definition at line 596 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                 {
    duk_small_int_t t;
 
    /* Here 'x' is a Unicode codepoint */
    if (DUK_LIKELY(x >= 0 && x <= 0xff)) {
        t = duk_hex_dectab[x];
        if (DUK_LIKELY(t >= 0)) {
            return t;
        }
    }
 
    /* Throwing an error this deep makes the error rather vague, but
     * saves hundreds of bytes of code.
     */
    DUK_ERROR_SYNTAX(lex_ctx->thr, "decode error");
    return 0;
}

References DUK_ERROR_SYNTAX, duk_hex_dectab, DUK_LIKELY, and duk_lexer_ctx::thr.

Referenced by duk__decode_hexesc_from_window(), duk__decode_uniesc_from_window(), duk_lexer_parse_js_input_element(), and duk_lexer_parse_re_token().

duk__init_lexer_window()

DUK_LOCAL void duk__init_lexer_window

(

duk_lexer_ctx *

lex_ctx

)

Definition at line 345 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                              {
    lex_ctx->window = lex_ctx->buffer;
    duk__fill_lexer_buffer(lex_ctx, 0);
}

References duk_lexer_ctx::buffer, duk__fill_lexer_buffer(), and duk_lexer_ctx::window.

Referenced by duk_lexer_setpoint().

duk__initbuffer()

DUK_LOCAL void duk__initbuffer

(

duk_lexer_ctx *

lex_ctx

)

Definition at line 512 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                       {
    /* Reuse buffer as is unless buffer has grown large. */
    if (DUK_HBUFFER_DYNAMIC_GET_SIZE(lex_ctx->buf) < DUK_LEXER_TEMP_BUF_LIMIT) {
        /* Keep current size */
    } else {
        duk_hbuffer_resize(lex_ctx->thr, lex_ctx->buf, DUK_LEXER_TEMP_BUF_LIMIT);
    }
 
    DUK_BW_INIT_WITHBUF(lex_ctx->thr, &lex_ctx->bw, lex_ctx->buf);
}

References duk_lexer_ctx::buf, duk_lexer_ctx::bw, DUK_BW_INIT_WITHBUF, DUK_HBUFFER_DYNAMIC_GET_SIZE, duk_hbuffer_resize(), DUK_LEXER_TEMP_BUF_LIMIT, and duk_lexer_ctx::thr.

duk__internbuffer()

DUK_LOCAL void duk__internbuffer	(	duk_lexer_ctx *	lex_ctx,
		duk_idx_t	valstack_idx )

Definition at line 550 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                 {
    duk_context *ctx = (duk_context *) lex_ctx->thr;
 
    DUK_ASSERT(valstack_idx == lex_ctx->slot1_idx || valstack_idx == lex_ctx->slot2_idx);
 
    DUK_BW_PUSH_AS_STRING(lex_ctx->thr, &lex_ctx->bw);
    duk_replace(ctx, valstack_idx);
}

References duk_lexer_ctx::bw, DUK_ASSERT, DUK_BW_PUSH_AS_STRING, duk_replace(), duk_lexer_ctx::slot1_idx, duk_lexer_ctx::slot2_idx, and duk_lexer_ctx::thr.

Referenced by duk_lexer_parse_js_input_element().

duk__is_hex_digit()

DUK_LOCAL duk_bool_t duk__is_hex_digit

(

duk_codepoint_t

x

)

Definition at line 615 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                          {
    if (DUK_LIKELY(x >= 0 && x <= 0xff)) {
        return (duk_hex_dectab[x] >= 0);
    }
    return 0;
}

References duk_hex_dectab, and DUK_LIKELY.

duk_lexer_initctx()

DUK_INTERNAL void duk_lexer_initctx

(

duk_lexer_ctx *

lex_ctx

)

Definition at line 563 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                            {
    DUK_ASSERT(lex_ctx != NULL);
 
    DUK_MEMZERO(lex_ctx, sizeof(*lex_ctx));
#if defined(DUK_USE_EXPLICIT_NULL_INIT)
#if defined(DUK_USE_LEXER_SLIDING_WINDOW)
    lex_ctx->window = NULL;
#endif
    lex_ctx->thr = NULL;
    lex_ctx->input = NULL;
    lex_ctx->buf = NULL;
#endif
}

References duk_lexer_ctx::buf, DUK_ASSERT, DUK_MEMZERO, duk_lexer_ctx::input, NULL, duk_lexer_ctx::thr, and duk_lexer_ctx::window.

duk_lexer_parse_js_input_element()

DUK_INTERNAL void duk_lexer_parse_js_input_element	(	duk_lexer_ctx *	lex_ctx,
		duk_token *	out_token,
		duk_bool_t	strict_mode,
		duk_bool_t	regexp_mode )

Definition at line 701 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                              {
    duk_codepoint_t x;           /* temporary, must be signed and 32-bit to hold Unicode code points */
    duk_small_uint_t advtok = 0; /* (advance << 8) + token_type, updated at function end,
                                  * init is unnecessary but suppresses "may be used uninitialized" warnings.
                                  */
    duk_bool_t got_lineterm = 0;  /* got lineterm preceding non-whitespace, non-lineterm token */
 
    if (++lex_ctx->token_count >= lex_ctx->token_limit) {
        DUK_ERROR_RANGE(lex_ctx->thr, "token limit");
        return;  /* unreachable */
    }
 
    out_token->t = DUK_TOK_EOF;
    out_token->t_nores = -1;  /* marker: copy t if not changed */
#if 0  /* not necessary to init, disabled for faster parsing */
    out_token->num = DUK_DOUBLE_NAN;
    out_token->str1 = NULL;
    out_token->str2 = NULL;
#endif
    out_token->num_escapes = 0;
    /* out_token->lineterm set by caller */
 
    /* This would be nice, but parsing is faster without resetting the
     * value slots.  The only side effect is that references to temporary
     * string values may linger until lexing is finished; they're then
     * freed normally.
     */
#if 0
    duk_to_undefined((duk_context *) lex_ctx->thr, lex_ctx->slot1_idx);
    duk_to_undefined((duk_context *) lex_ctx->thr, lex_ctx->slot2_idx);
#endif
 
    /* 'advtok' indicates how much to advance and which token id to assign
     * at the end.  This shared functionality minimizes code size.  All
     * code paths are required to set 'advtok' to some value, so no default
     * init value is used.  Code paths calling DUK_ERROR() never return so
     * they don't need to set advtok.
     */
 
    /*
     *  Matching order:
     *
     *    Punctuator first chars, also covers comments, regexps
     *    LineTerminator
     *    Identifier or reserved word, also covers null/true/false literals
     *    NumericLiteral
     *    StringLiteral
     *    EOF
     *
     *  The order does not matter as long as the longest match is
     *  always correctly identified.  There are order dependencies
     *  in the clauses, so it's not trivial to convert to a switch.
     */
 
 restart_lineupdate:
    out_token->start_line = lex_ctx->window[0].line;
 
 restart:
    out_token->start_offset = lex_ctx->window[0].offset;
 
    x = DUK__L0();
 
    switch (x) {
    case DUK_ASC_SPACE:
    case DUK_ASC_HT:  /* fast paths for space and tab */
        DUK__ADVANCECHARS(lex_ctx, 1);
        goto restart;
    case DUK_ASC_LF:  /* LF line terminator; CR LF and Unicode lineterms are handled in slow path */
        DUK__ADVANCECHARS(lex_ctx, 1);
        got_lineterm = 1;
        goto restart_lineupdate;
    case DUK_ASC_SLASH:  /* '/' */
        if (DUK__L1() == '/') {
            /*
             *  E5 Section 7.4, allow SourceCharacter (which is any 16-bit
             *  code point).
             */
 
            /* DUK__ADVANCECHARS(lex_ctx, 2) would be correct here, but it unnecessary */
            for (;;) {
                x = DUK__L0();
                if (x < 0 || duk_unicode_is_line_terminator(x)) {
                    break;
                }
                DUK__ADVANCECHARS(lex_ctx, 1);
            }
            goto restart;  /* line terminator will be handled on next round */
        } else if (DUK__L1() == '*') {
            /*
             *  E5 Section 7.4.  If the multi-line comment contains a newline,
             *  it is treated like a single line terminator for automatic
             *  semicolon insertion.
             */
 
            duk_bool_t last_asterisk = 0;
            DUK__ADVANCECHARS(lex_ctx, 2);
            for (;;) {
                x = DUK__L0();
                if (x < 0) {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "eof in multiline comment");
                }
                DUK__ADVANCECHARS(lex_ctx, 1);
                if (last_asterisk && x == '/') {
                    break;
                }
                if (duk_unicode_is_line_terminator(x)) {
                    got_lineterm = 1;
                }
                last_asterisk = (x == '*');
            }
            goto restart_lineupdate;
        } else if (regexp_mode) {
#if defined(DUK_USE_REGEXP_SUPPORT)
            /*
             *  "/" followed by something in regexp mode.  See E5 Section 7.8.5.
             *
             *  RegExp parsing is a bit complex.  First, the regexp body is delimited
             *  by forward slashes, but the body may also contain forward slashes as
             *  part of an escape sequence or inside a character class (delimited by
             *  square brackets).  A mini state machine is used to implement these.
             *
             *  Further, an early (parse time) error must be thrown if the regexp
             *  would cause a run-time error when used in the expression new RegExp(...).
             *  Parsing here simply extracts the (candidate) regexp, and also accepts
             *  invalid regular expressions (which are delimited properly).  The caller
             *  (compiler) must perform final validation and regexp compilation.
             *
             *  RegExp first char may not be '/' (single line comment) or '*' (multi-
             *  line comment).  These have already been checked above, so there is no
             *  need below for special handling of the first regexp character as in
             *  the E5 productions.
             *
             *  About unicode escapes within regexp literals:
             *
             *      E5 Section 7.8.5 grammar does NOT accept \uHHHH escapes.
             *      However, Section 6 states that regexps accept the escapes,
             *      see paragraph starting with "In string literals...".
             *      The regexp grammar, which sees the decoded regexp literal
             *      (after lexical parsing) DOES have a \uHHHH unicode escape.
             *      So, for instance:
             *
             *          /\u1234/
             *
             *      should first be parsed by the lexical grammar as:
             *
             *          '\' 'u'      RegularExpressionBackslashSequence
             *          '1'          RegularExpressionNonTerminator
             *          '2'          RegularExpressionNonTerminator
             *          '3'          RegularExpressionNonTerminator
             *          '4'          RegularExpressionNonTerminator
             *
             *      and the escape itself is then parsed by the regexp engine.
             *      This is the current implementation.
             *
             *  Minor spec inconsistency:
             *
             *      E5 Section 7.8.5 RegularExpressionBackslashSequence is:
             *
             *         \ RegularExpressionNonTerminator
             *
             *      while Section A.1 RegularExpressionBackslashSequence is:
             *
             *         \ NonTerminator
             *
             *      The latter is not normative and a typo.
             *
             */
 
            /* first, parse regexp body roughly */
 
            duk_small_int_t state = 0;  /* 0=base, 1=esc, 2=class, 3=class+esc */
 
            DUK__INITBUFFER(lex_ctx);
            for (;;) {
                DUK__ADVANCECHARS(lex_ctx, 1);  /* skip opening slash on first loop */
                x = DUK__L0();
                if (x < 0 || duk_unicode_is_line_terminator(x)) {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "eof or line terminator in regexp");
                }
                x = DUK__L0();  /* re-read to avoid spill / fetch */
                if (state == 0) {
                    if (x == '/') {
                        DUK__ADVANCECHARS(lex_ctx, 1);  /* eat closing slash */
                        break;
                    } else if (x == '\\') {
                        state = 1;
                    } else if (x == '[') {
                        state = 2;
                    }
                } else if (state == 1) {
                    state = 0;
                } else if (state == 2) {
                    if (x == ']') {
                        state = 0;
                    } else if (x == '\\') {
                        state = 3;
                    }
                } else { /* state == 3 */
                    state = 2;
                }
                DUK__APPENDBUFFER(lex_ctx, x);
            }
            duk__internbuffer(lex_ctx, lex_ctx->slot1_idx);
            out_token->str1 = duk_get_hstring((duk_context *) lex_ctx->thr, lex_ctx->slot1_idx);
 
            /* second, parse flags */
 
            DUK__INITBUFFER(lex_ctx);
            for (;;) {
                x = DUK__L0();
                if (!duk_unicode_is_identifier_part(x)) {
                    break;
                }
                x = DUK__L0();  /* re-read to avoid spill / fetch */
                DUK__APPENDBUFFER(lex_ctx, x);
                DUK__ADVANCECHARS(lex_ctx, 1);
            }
            duk__internbuffer(lex_ctx, lex_ctx->slot2_idx);
            out_token->str2 = duk_get_hstring((duk_context *) lex_ctx->thr, lex_ctx->slot2_idx);
 
            DUK__INITBUFFER(lex_ctx);  /* free some memory */
 
            /* validation of the regexp is caller's responsibility */
 
            advtok = DUK__ADVTOK(0, DUK_TOK_REGEXP);
#else
            DUK_ERROR_SYNTAX(lex_ctx->thr, "regexp support disabled");
#endif
        } else if (DUK__L1() == '=') {
            /* "/=" and not in regexp mode */
            advtok = DUK__ADVTOK(2, DUK_TOK_DIV_EQ);
        } else {
            /* "/" and not in regexp mode */
            advtok = DUK__ADVTOK(1, DUK_TOK_DIV);
        }
        break;
    case DUK_ASC_LCURLY:  /* '{' */
        advtok = DUK__ADVTOK(1, DUK_TOK_LCURLY);
        break;
    case DUK_ASC_RCURLY:  /* '}' */
        advtok = DUK__ADVTOK(1, DUK_TOK_RCURLY);
        break;
    case DUK_ASC_LPAREN:  /* '(' */
        advtok = DUK__ADVTOK(1, DUK_TOK_LPAREN);
        break;
    case DUK_ASC_RPAREN:  /* ')' */
        advtok = DUK__ADVTOK(1, DUK_TOK_RPAREN);
        break;
    case DUK_ASC_LBRACKET:  /* '[' */
        advtok = DUK__ADVTOK(1, DUK_TOK_LBRACKET);
        break;
    case DUK_ASC_RBRACKET:  /* ']' */
        advtok = DUK__ADVTOK(1, DUK_TOK_RBRACKET);
        break;
    case DUK_ASC_PERIOD:  /* '.' */
        if (DUK__ISDIGIT(DUK__L1())) {
            /* Period followed by a digit can only start DecimalLiteral
             * (handled in slow path).  We could jump straight into the
             * DecimalLiteral handling but should avoid goto to inside
             * a block.
             */
            goto slow_path;
        }
        advtok = DUK__ADVTOK(1, DUK_TOK_PERIOD);
        break;
    case DUK_ASC_SEMICOLON:  /* ';' */
        advtok = DUK__ADVTOK(1, DUK_TOK_SEMICOLON);
        break;
    case DUK_ASC_COMMA:  /* ',' */
        advtok = DUK__ADVTOK(1, DUK_TOK_COMMA);
        break;
    case DUK_ASC_LANGLE:  /* '<' */
        if (DUK__L1() == '<' && DUK__L2() == '=') {
            advtok = DUK__ADVTOK(3, DUK_TOK_ALSHIFT_EQ);
        } else if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_LE);
        } else if (DUK__L1() == '<') {
            advtok = DUK__ADVTOK(2, DUK_TOK_ALSHIFT);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_LT);
        }
        break;
    case DUK_ASC_RANGLE:  /* '>' */
        if (DUK__L1() == '>' && DUK__L2() == '>' && DUK__L3() == '=') {
            advtok = DUK__ADVTOK(4, DUK_TOK_RSHIFT_EQ);
        } else if (DUK__L1() == '>' && DUK__L2() == '>') {
            advtok = DUK__ADVTOK(3, DUK_TOK_RSHIFT);
        } else if (DUK__L1() == '>' && DUK__L2() == '=') {
            advtok = DUK__ADVTOK(3, DUK_TOK_ARSHIFT_EQ);
        } else if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_GE);
        } else if (DUK__L1() == '>') {
            advtok = DUK__ADVTOK(2, DUK_TOK_ARSHIFT);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_GT);
        }
        break;
    case DUK_ASC_EQUALS:  /* '=' */
        if (DUK__L1() == '=' && DUK__L2() == '=') {
            advtok = DUK__ADVTOK(3, DUK_TOK_SEQ);
        } else if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_EQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_EQUALSIGN);
        }
        break;
    case DUK_ASC_EXCLAMATION:  /* '!' */
        if (DUK__L1() == '=' && DUK__L2() == '=') {
            advtok = DUK__ADVTOK(3, DUK_TOK_SNEQ);
        } else if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_NEQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_LNOT);
        }
        break;
    case DUK_ASC_PLUS:  /* '+' */
        if (DUK__L1() == '+') {
            advtok = DUK__ADVTOK(2, DUK_TOK_INCREMENT);
        } else if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_ADD_EQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_ADD);
        }
        break;
    case DUK_ASC_MINUS:  /* '-' */
        if (DUK__L1() == '-') {
            advtok = DUK__ADVTOK(2, DUK_TOK_DECREMENT);
        } else if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_SUB_EQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_SUB);
        }
        break;
    case DUK_ASC_STAR:  /* '*' */
        if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_MUL_EQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_MUL);
        }
        break;
    case DUK_ASC_PERCENT:  /* '%' */
        if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_MOD_EQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_MOD);
        }
        break;
    case DUK_ASC_AMP:  /* '&' */
        if (DUK__L1() == '&') {
            advtok = DUK__ADVTOK(2, DUK_TOK_LAND);
        } else if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_BAND_EQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_BAND);
        }
        break;
    case DUK_ASC_PIPE:  /* '|' */
        if (DUK__L1() == '|') {
            advtok = DUK__ADVTOK(2, DUK_TOK_LOR);
        } else if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_BOR_EQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_BOR);
        }
        break;
    case DUK_ASC_CARET:  /* '^' */
        if (DUK__L1() == '=') {
            advtok = DUK__ADVTOK(2, DUK_TOK_BXOR_EQ);
        } else {
            advtok = DUK__ADVTOK(1, DUK_TOK_BXOR);
        }
        break;
    case DUK_ASC_TILDE:  /* '~' */
        advtok = DUK__ADVTOK(1, DUK_TOK_BNOT);
        break;
    case DUK_ASC_QUESTION:  /* '?' */
        advtok = DUK__ADVTOK(1, DUK_TOK_QUESTION);
        break;
    case DUK_ASC_COLON:  /* ':' */
        advtok = DUK__ADVTOK(1, DUK_TOK_COLON);
        break;
    case DUK_ASC_DOUBLEQUOTE:    /* '"' */
    case DUK_ASC_SINGLEQUOTE: {  /* '\'' */
        duk_small_int_t quote = x;  /* Note: duk_uint8_t type yields larger code */
        duk_small_int_t adv;
 
        DUK__INITBUFFER(lex_ctx);
        for (;;) {
            DUK__ADVANCECHARS(lex_ctx, 1);  /* eat opening quote on first loop */
            x = DUK__L0();
            if (x < 0 || duk_unicode_is_line_terminator(x)) {
                DUK_ERROR_SYNTAX(lex_ctx->thr, "eof or line terminator in string literal");
            }
            if (x == quote) {
                DUK__ADVANCECHARS(lex_ctx, 1);  /* eat closing quote */
                break;
            }
            if (x == '\\') {
                /* DUK__L0        -> '\' char
                 * DUK__L1 ... DUK__L5 -> more lookup
                 */
 
                x = DUK__L1();
 
                /* How much to advance before next loop; note that next loop
                 * will advance by 1 anyway, so -1 from the total escape
                 * length (e.g. len('\uXXXX') - 1 = 6 - 1).  As a default,
                 * 1 is good.
                 */
                adv = 2 - 1;  /* note: long live range */
 
                if (x < 0) {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "eof or line terminator in string literal");
                }
                if (duk_unicode_is_line_terminator(x)) {
                    /* line continuation */
                    if (x == 0x000d && DUK__L2() == 0x000a) {
                        /* CR LF again a special case */
                        adv = 3 - 1;
                    }
                } else if (x == '\'') {
                    DUK__APPENDBUFFER(lex_ctx, 0x0027);
                } else if (x == '"') {
                    DUK__APPENDBUFFER(lex_ctx, 0x0022);
                } else if (x == '\\') {
                    DUK__APPENDBUFFER(lex_ctx, 0x005c);
                } else if (x == 'b') {
                    DUK__APPENDBUFFER(lex_ctx, 0x0008);
                } else if (x == 'f') {
                    DUK__APPENDBUFFER(lex_ctx, 0x000c);
                } else if (x == 'n') {
                    DUK__APPENDBUFFER(lex_ctx, 0x000a);
                } else if (x == 'r') {
                    DUK__APPENDBUFFER(lex_ctx, 0x000d);
                } else if (x == 't') {
                    DUK__APPENDBUFFER(lex_ctx, 0x0009);
                } else if (x == 'v') {
                    DUK__APPENDBUFFER(lex_ctx, 0x000b);
                } else if (x == 'x') {
                    adv = 4 - 1;
                    DUK__APPENDBUFFER(lex_ctx, duk__decode_hexesc_from_window(lex_ctx, 2));
                } else if (x == 'u') {
                    adv = 6 - 1;
                    DUK__APPENDBUFFER(lex_ctx, duk__decode_uniesc_from_window(lex_ctx, 2));
                } else if (DUK__ISDIGIT(x)) {
                    duk_codepoint_t ch = 0;  /* initialized to avoid warnings of unused var */
 
                    /*
                     *  Octal escape or zero escape:
                     *    \0                                     (lookahead not DecimalDigit)
                     *    \1 ... \7                              (lookahead not DecimalDigit)
                     *    \ZeroToThree OctalDigit                (lookahead not DecimalDigit)
                     *    \FourToSeven OctalDigit                (no lookahead restrictions)
                     *    \ZeroToThree OctalDigit OctalDigit     (no lookahead restrictions)
                     *
                     *  Zero escape is part of the standard syntax.  Octal escapes are
                     *  defined in E5 Section B.1.2, and are only allowed in non-strict mode.
                     *  Any other productions starting with a decimal digit are invalid.
                     */
 
                    if (x == '0' && !DUK__ISDIGIT(DUK__L2())) {
                        /* Zero escape (also allowed in non-strict mode) */
                        ch = 0;
                        /* adv = 2 - 1 default OK */
#if defined(DUK_USE_OCTAL_SUPPORT)
                    } else if (strict_mode) {
                        /* No other escape beginning with a digit in strict mode */
                        DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid escape in string literal");
                    } else if (DUK__ISDIGIT03(x) && DUK__ISOCTDIGIT(DUK__L2()) && DUK__ISOCTDIGIT(DUK__L3())) {
                        /* Three digit octal escape, digits validated. */
                        adv = 4 - 1;
                        ch = (duk__hexval(lex_ctx, x) << 6) +
                             (duk__hexval(lex_ctx, DUK__L2()) << 3) +
                             duk__hexval(lex_ctx, DUK__L3());
                    } else if (((DUK__ISDIGIT03(x) && !DUK__ISDIGIT(DUK__L3())) || DUK__ISDIGIT47(x)) &&
                               DUK__ISOCTDIGIT(DUK__L2())) {
                        /* Two digit octal escape, digits validated.
                         *
                         * The if-condition is a bit tricky.  We could catch e.g.
                         * '\039' in the three-digit escape and fail it there (by
                             * validating the digits), but we want to avoid extra
                         * additional validation code.
                         */
                        adv = 3 - 1;
                        ch = (duk__hexval(lex_ctx, x) << 3) +
                             duk__hexval(lex_ctx, DUK__L2());
                    } else if (DUK__ISDIGIT(x) && !DUK__ISDIGIT(DUK__L2())) {
                        /* One digit octal escape, digit validated. */
                        /* adv = 2 default OK */
                        ch = duk__hexval(lex_ctx, x);
#else
                    /* fall through to error */
#endif
                    } else {
                        DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid escape in string literal");
                    }
 
                    DUK__APPENDBUFFER(lex_ctx, ch);
                } else {
                    /* escaped NonEscapeCharacter */
                    DUK__APPENDBUFFER(lex_ctx, x);
                }
                DUK__ADVANCECHARS(lex_ctx, adv);
 
                /* Track number of escapes; count not really needed but directive
                 * prologues need to detect whether there were any escapes or line
                 * continuations or not.
                 */
                out_token->num_escapes++;
            } else {
                /* part of string */
                DUK__APPENDBUFFER(lex_ctx, x);
            }
        }
 
        duk__internbuffer(lex_ctx, lex_ctx->slot1_idx);
        out_token->str1 = duk_get_hstring((duk_context *) lex_ctx->thr, lex_ctx->slot1_idx);
 
        DUK__INITBUFFER(lex_ctx);  /* free some memory */
 
        advtok = DUK__ADVTOK(0, DUK_TOK_STRING);
        break;
    }
    default:
        goto slow_path;
    }  /* switch */
 
    goto skip_slow_path;
 
 slow_path:
    if (duk_unicode_is_line_terminator(x)) {
        if (x == 0x000d && DUK__L1() == 0x000a) {
            /*
             *  E5 Section 7.3: CR LF is detected as a single line terminator for
             *  line numbers.  Here we also detect it as a single line terminator
             *  token.
             */
            DUK__ADVANCECHARS(lex_ctx, 2);
        } else {
            DUK__ADVANCECHARS(lex_ctx, 1);
        }
        got_lineterm = 1;
        goto restart_lineupdate;
    } else if (duk_unicode_is_identifier_start(x) || x == '\\') {
        /*
         *  Parse an identifier and then check whether it is:
         *    - reserved word (keyword or other reserved word)
         *    - "null"  (NullLiteral)
         *    - "true"  (BooleanLiteral)
         *    - "false" (BooleanLiteral)
         *    - anything else => identifier
         *
         *  This does not follow the E5 productions cleanly, but is
         *  useful and compact.
         *
         *  Note that identifiers may contain Unicode escapes,
         *  see E5 Sections 6 and 7.6.  They must be decoded first,
         *  and the result checked against allowed characters.
         *  The above if-clause accepts an identifier start and an
         *  '\' character -- no other token can begin with a '\'.
         *
         *  Note that "get" and "set" are not reserved words in E5
         *  specification so they are recognized as plain identifiers
         *  (the tokens DUK_TOK_GET and DUK_TOK_SET are actually not
         *  used now).  The compiler needs to work around this.
         *
         *  Strictly speaking, following Ecmascript longest match
         *  specification, an invalid escape for the first character
         *  should cause a syntax error.  However, an invalid escape
         *  for IdentifierParts should just terminate the identifier
         *  early (longest match), and let the next tokenization
         *  fail.  For instance Rhino croaks with 'foo\z' when
         *  parsing the identifier.  This has little practical impact.
         */
 
        duk_small_int_t i, i_end;
        duk_bool_t first = 1;
        duk_hstring *str;
 
        DUK__INITBUFFER(lex_ctx);
        for (;;) {
            /* re-lookup first char on first loop */
            if (DUK__L0() == '\\') {
                duk_codepoint_t ch;
                if (DUK__L1() != 'u') {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid unicode escape in identifier");
                }
 
                ch = duk__decode_uniesc_from_window(lex_ctx, 2);
 
                /* IdentifierStart is stricter than IdentifierPart, so if the first
                 * character is escaped, must have a stricter check here.
                 */
                if (!(first ? duk_unicode_is_identifier_start(ch) : duk_unicode_is_identifier_part(ch))) {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid unicode escape in identifier");
                }
                DUK__APPENDBUFFER(lex_ctx, ch);
                DUK__ADVANCECHARS(lex_ctx, 6);
 
                /* Track number of escapes: necessary for proper keyword
                 * detection.
                 */
                out_token->num_escapes++;
            } else {
                /* Note: first character is checked against this.  But because
                 * IdentifierPart includes all IdentifierStart characters, and
                 * the first character (if unescaped) has already been checked
                 * in the if condition, this is OK.
                 */
                if (!duk_unicode_is_identifier_part(DUK__L0())) {
                    break;
                }
                DUK__APPENDBUFFER(lex_ctx, DUK__L0());
                DUK__ADVANCECHARS(lex_ctx, 1);
            }
            first = 0;
        }
 
        duk__internbuffer(lex_ctx, lex_ctx->slot1_idx);
        out_token->str1 = duk_get_hstring((duk_context *) lex_ctx->thr, lex_ctx->slot1_idx);
        str = out_token->str1;
        DUK_ASSERT(str != NULL);
        out_token->t_nores = DUK_TOK_IDENTIFIER;
 
        DUK__INITBUFFER(lex_ctx);  /* free some memory */
 
        /*
         *  Interned identifier is compared against reserved words, which are
         *  currently interned into the heap context.  See genbuiltins.py.
         *
         *  Note that an escape in the identifier disables recognition of
         *  keywords; e.g. "\u0069f = 1;" is a valid statement (assigns to
         *  identifier named "if").  This is not necessarily compliant,
         *  see test-dec-escaped-char-in-keyword.js.
         *
         *  Note: "get" and "set" are awkward.  They are not officially
         *  ReservedWords (and indeed e.g. "var set = 1;" is valid), and
         *  must come out as DUK_TOK_IDENTIFIER.  The compiler needs to
         *  work around this a bit.
         */
 
        /* XXX: optimize by adding the token numbers directly into the
         * always interned duk_hstring objects (there should be enough
         * flag bits free for that)?
         */
 
        i_end = (strict_mode ? DUK_STRIDX_END_RESERVED : DUK_STRIDX_START_STRICT_RESERVED);
 
        advtok = DUK__ADVTOK(0, DUK_TOK_IDENTIFIER);
        if (out_token->num_escapes == 0) {
            for (i = DUK_STRIDX_START_RESERVED; i < i_end; i++) {
                DUK_ASSERT(i >= 0 && i < DUK_HEAP_NUM_STRINGS);
                if (DUK_HTHREAD_GET_STRING(lex_ctx->thr, i) == str) {
                    advtok = DUK__ADVTOK(0, DUK_STRIDX_TO_TOK(i));
                    break;
                }
            }
        }
    } else if (DUK__ISDIGIT(x) || (x == '.')) {
        /* Note: decimal number may start with a period, but must be followed by a digit */
 
        /*
         *  DecimalLiteral, HexIntegerLiteral, OctalIntegerLiteral
         *  "pre-parsing", followed by an actual, accurate parser step.
         *
         *  Note: the leading sign character ('+' or '-') is -not- part of
         *  the production in E5 grammar, and that the a DecimalLiteral
         *  starting with a '0' must be followed by a non-digit.  Leading
         *  zeroes are syntax errors and must be checked for.
         *
         *  XXX: the two step parsing process is quite awkward, it would
         *  be more straightforward to allow numconv to parse the longest
         *  valid prefix (it already does that, it only needs to indicate
         *  where the input ended).  However, the lexer decodes characters
         *  using a lookup window, so this is not a trivial change.
         */
 
        /* XXX: because of the final check below (that the literal is not
         * followed by a digit), this could maybe be simplified, if we bail
         * out early from a leading zero (and if there are no periods etc).
         * Maybe too complex.
         */
 
        duk_double_t val;
        duk_bool_t int_only = 0;
        duk_bool_t allow_hex = 0;
        duk_small_int_t state;  /* 0=before period/exp,
                                 * 1=after period, before exp
                                 * 2=after exp, allow '+' or '-'
                                 * 3=after exp and exp sign
                                 */
        duk_small_uint_t s2n_flags;
        duk_codepoint_t y;
 
        DUK__INITBUFFER(lex_ctx);
        y = DUK__L1();
        if (x == '0' && (y == 'x' || y == 'X')) {
            DUK__APPENDBUFFER(lex_ctx, x);
            DUK__APPENDBUFFER(lex_ctx, y);
            DUK__ADVANCECHARS(lex_ctx, 2);
            int_only = 1;
            allow_hex = 1;
#if defined(DUK_USE_OCTAL_SUPPORT)
        } else if (!strict_mode && x == '0' && DUK__ISDIGIT(y)) {
            /* Note: if DecimalLiteral starts with a '0', it can only be
             * followed by a period or an exponent indicator which starts
             * with 'e' or 'E'.  Hence the if-check above ensures that
             * OctalIntegerLiteral is the only valid NumericLiteral
             * alternative at this point (even if y is, say, '9').
             */
 
            DUK__APPENDBUFFER(lex_ctx, x);
            DUK__ADVANCECHARS(lex_ctx, 1);
            int_only = 1;
#endif
        }
 
        state = 0;
        for (;;) {
            x = DUK__L0();  /* re-lookup curr char on first round */
            if (DUK__ISDIGIT(x)) {
                /* Note: intentionally allow leading zeroes here, as the
                 * actual parser will check for them.
                 */
                if (state == 2) {
                    state = 3;
                }
            } else if (allow_hex && DUK__ISHEXDIGIT(x)) {
                /* Note: 'e' and 'E' are also accepted here. */
                ;
            } else if (x == '.') {
                if (state >= 1 || int_only) {
                    break;
                } else {
                    state = 1;
                }
            } else if (x == 'e' || x == 'E') {
                if (state >= 2 || int_only) {
                    break;
                } else {
                    state = 2;
                }
            } else if (x == '-' || x == '+') {
                if (state != 2) {
                    break;
                } else {
                    state = 3;
                }
            } else {
                break;
            }
            DUK__APPENDBUFFER(lex_ctx, x);
            DUK__ADVANCECHARS(lex_ctx, 1);
        }
 
        /* XXX: better coercion */
        duk__internbuffer(lex_ctx, lex_ctx->slot1_idx);
 
        s2n_flags = DUK_S2N_FLAG_ALLOW_EXP |
                    DUK_S2N_FLAG_ALLOW_FRAC |
                    DUK_S2N_FLAG_ALLOW_NAKED_FRAC |
                    DUK_S2N_FLAG_ALLOW_EMPTY_FRAC |
#if defined(DUK_USE_OCTAL_SUPPORT)
                    (strict_mode ? 0 : DUK_S2N_FLAG_ALLOW_AUTO_OCT_INT) |
#endif
                    DUK_S2N_FLAG_ALLOW_AUTO_HEX_INT;
 
        duk_dup((duk_context *) lex_ctx->thr, lex_ctx->slot1_idx);
        duk_numconv_parse((duk_context *) lex_ctx->thr, 10 /*radix*/, s2n_flags);
        val = duk_to_number((duk_context *) lex_ctx->thr, -1);
        if (DUK_ISNAN(val)) {
            DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid numeric literal");
        }
        duk_replace((duk_context *) lex_ctx->thr, lex_ctx->slot1_idx);  /* could also just pop? */
 
        DUK__INITBUFFER(lex_ctx);  /* free some memory */
 
        /* Section 7.8.3 (note): NumericLiteral must be followed by something other than
         * IdentifierStart or DecimalDigit.
         */
 
        if (DUK__ISDIGIT(DUK__L0()) || duk_unicode_is_identifier_start(DUK__L0())) {
            DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid numeric literal");
        }
 
        out_token->num = val;
        advtok = DUK__ADVTOK(0, DUK_TOK_NUMBER);
    } else if (duk_unicode_is_whitespace(DUK__LOOKUP(lex_ctx, 0))) {
        DUK__ADVANCECHARS(lex_ctx, 1);
        goto restart;
    } else if (x < 0) {
        advtok = DUK__ADVTOK(0, DUK_TOK_EOF);
    } else {
        DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid token");
    }
 skip_slow_path:
 
    /*
     *  Shared exit path
     */
 
    DUK__ADVANCEBYTES(lex_ctx, advtok >> 8);
    out_token->t = advtok & 0xff;
    if (out_token->t_nores < 0) {
        out_token->t_nores = out_token->t;
    }
    out_token->lineterm = got_lineterm;
 
    /* Automatic semicolon insertion is allowed if a token is preceded
     * by line terminator(s), or terminates a statement list (right curly
     * or EOF).
     */
    if (got_lineterm || out_token->t == DUK_TOK_RCURLY || out_token->t == DUK_TOK_EOF) {
        out_token->allow_auto_semi = 1;
    } else {
        out_token->allow_auto_semi = 0;
    }
}

References duk_token::allow_auto_semi, DUK__ADVANCEBYTES, DUK__ADVANCECHARS, DUK__ADVTOK, DUK__APPENDBUFFER, duk__decode_hexesc_from_window(), duk__decode_uniesc_from_window(), duk__hexval(), DUK__INITBUFFER, duk__internbuffer(), DUK__ISDIGIT, DUK__ISDIGIT03, DUK__ISDIGIT47, DUK__ISHEXDIGIT, DUK__ISOCTDIGIT, DUK__L0, DUK__L1, DUK__L2, DUK__L3, DUK__LOOKUP, DUK_ASC_AMP, DUK_ASC_CARET, DUK_ASC_COLON, DUK_ASC_COMMA, DUK_ASC_DOUBLEQUOTE, DUK_ASC_EQUALS, DUK_ASC_EXCLAMATION, DUK_ASC_HT, DUK_ASC_LANGLE, DUK_ASC_LBRACKET, DUK_ASC_LCURLY, DUK_ASC_LF, DUK_ASC_LPAREN, DUK_ASC_MINUS, DUK_ASC_PERCENT, DUK_ASC_PERIOD, DUK_ASC_PIPE, DUK_ASC_PLUS, DUK_ASC_QUESTION, DUK_ASC_RANGLE, DUK_ASC_RBRACKET, DUK_ASC_RCURLY, DUK_ASC_RPAREN, DUK_ASC_SEMICOLON, DUK_ASC_SINGLEQUOTE, DUK_ASC_SLASH, DUK_ASC_SPACE, DUK_ASC_STAR, DUK_ASC_TILDE, DUK_ASSERT, DUK_DOUBLE_NAN, duk_dup(), DUK_ERROR_RANGE, DUK_ERROR_SYNTAX, duk_get_hstring(), DUK_HEAP_NUM_STRINGS, DUK_HTHREAD_GET_STRING, DUK_ISNAN, duk_numconv_parse(), duk_replace(), DUK_S2N_FLAG_ALLOW_AUTO_HEX_INT, DUK_S2N_FLAG_ALLOW_AUTO_OCT_INT, DUK_S2N_FLAG_ALLOW_EMPTY_FRAC, DUK_S2N_FLAG_ALLOW_EXP, DUK_S2N_FLAG_ALLOW_FRAC, DUK_S2N_FLAG_ALLOW_NAKED_FRAC, DUK_STRIDX_END_RESERVED, DUK_STRIDX_START_RESERVED, DUK_STRIDX_START_STRICT_RESERVED, DUK_STRIDX_TO_TOK, duk_to_number(), duk_to_undefined(), DUK_TOK_ADD, DUK_TOK_ADD_EQ, DUK_TOK_ALSHIFT, DUK_TOK_ALSHIFT_EQ, DUK_TOK_ARSHIFT, DUK_TOK_ARSHIFT_EQ, DUK_TOK_BAND, DUK_TOK_BAND_EQ, DUK_TOK_BNOT, DUK_TOK_BOR, DUK_TOK_BOR_EQ, DUK_TOK_BXOR, DUK_TOK_BXOR_EQ, DUK_TOK_COLON, DUK_TOK_COMMA, DUK_TOK_DECREMENT, DUK_TOK_DIV, DUK_TOK_DIV_EQ, DUK_TOK_EOF, DUK_TOK_EQ, DUK_TOK_EQUALSIGN, DUK_TOK_GE, DUK_TOK_GT, DUK_TOK_IDENTIFIER, DUK_TOK_INCREMENT, DUK_TOK_LAND, DUK_TOK_LBRACKET, DUK_TOK_LCURLY, DUK_TOK_LE, DUK_TOK_LNOT, DUK_TOK_LOR, DUK_TOK_LPAREN, DUK_TOK_LT, DUK_TOK_MOD, DUK_TOK_MOD_EQ, DUK_TOK_MUL, DUK_TOK_MUL_EQ, DUK_TOK_NEQ, DUK_TOK_NUMBER, DUK_TOK_PERIOD, DUK_TOK_QUESTION, DUK_TOK_RBRACKET, DUK_TOK_RCURLY, DUK_TOK_REGEXP, DUK_TOK_RPAREN, DUK_TOK_RSHIFT, DUK_TOK_RSHIFT_EQ, DUK_TOK_SEMICOLON, DUK_TOK_SEQ, DUK_TOK_SNEQ, DUK_TOK_STRING, DUK_TOK_SUB, DUK_TOK_SUB_EQ, duk_unicode_is_identifier_part(), duk_unicode_is_identifier_start(), duk_unicode_is_line_terminator(), duk_unicode_is_whitespace(), duk_lexer_codepoint::line, duk_token::lineterm, NULL, duk_token::num, duk_token::num_escapes, duk_lexer_codepoint::offset, duk_lexer_ctx::slot1_idx, duk_lexer_ctx::slot2_idx, duk_token::start_line, duk_token::start_offset, duk_token::str1, duk_token::str2, duk_token::t, duk_token::t_nores, duk_lexer_ctx::thr, duk_lexer_ctx::token_count, duk_lexer_ctx::token_limit, and duk_lexer_ctx::window.

duk_lexer_parse_re_ranges()

DUK_INTERNAL void duk_lexer_parse_re_ranges	(	duk_lexer_ctx *	lex_ctx,
		duk_re_range_callback	gen_range,
		void *	userdata )

Definition at line 1885 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                                                     {
    duk_codepoint_t start = -1;
    duk_codepoint_t ch;
    duk_codepoint_t x;
    duk_bool_t dash = 0;
 
    DUK_DD(DUK_DDPRINT("parsing regexp ranges"));
 
    for (;;) {
        x = DUK__L0();
        DUK__ADVANCECHARS(lex_ctx, 1);
 
        ch = -1;  /* not strictly necessary, but avoids "uninitialized variable" warnings */
        DUK_UNREF(ch);
 
        if (x < 0) {
            DUK_ERROR_SYNTAX(lex_ctx->thr, "eof in character class");
        } else if (x == ']') {
            DUK_ASSERT(!dash);  /* lookup should prevent this */
            if (start >= 0) {
                gen_range(userdata, start, start, 0);
            }
            break;
        } else if (x == '-') {
            if (start >= 0 && !dash && DUK__L0() != ']') {
                /* '-' as a range indicator */
                dash = 1;
                continue;
            } else {
                /* '-' verbatim */
                ch = x;
            }
        } else if (x == '\\') {
            /*
             *  The escapes are same as outside a character class, except that \b has a
             *  different meaning, and \B and backreferences are prohibited (see E5
             *  Section 15.10.2.19).  However, it's difficult to share code because we
             *  handle e.g. "\n" very differently: here we generate a single character
             *  range for it.
             */
 
            x = DUK__L0();
            DUK__ADVANCECHARS(lex_ctx, 1);
 
            if (x == 'b') {
                /* Note: '\b' in char class is different than outside (assertion),
                 * '\B' is not allowed and is caught by the duk_unicode_is_identifier_part()
                 * check below.
                 */
                ch = 0x0008;
            } else if (x == 'f') {
                ch = 0x000c;
            } else if (x == 'n') {
                ch = 0x000a;
            } else if (x == 't') {
                ch = 0x0009;
            } else if (x == 'r') {
                ch = 0x000d;
            } else if (x == 'v') {
                ch = 0x000b;
            } else if (x == 'c') {
                x = DUK__L0();
                DUK__ADVANCECHARS(lex_ctx, 1);
                if ((x >= 'a' && x <= 'z') ||
                    (x >= 'A' && x <= 'Z')) {
                    ch = (x % 32);
                } else {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp escape");
                    return;  /* never reached, but avoids warnings of
                              * potentially unused variables.
                              */
                }
            } else if (x == 'x') {
                ch = duk__decode_hexesc_from_window(lex_ctx, 0);
                DUK__ADVANCECHARS(lex_ctx, 2);
            } else if (x == 'u') {
                ch = duk__decode_uniesc_from_window(lex_ctx, 0);
                DUK__ADVANCECHARS(lex_ctx, 4);
            } else if (x == 'd') {
                duk__emit_u16_direct_ranges(lex_ctx,
                                            gen_range,
                                            userdata,
                                            duk_unicode_re_ranges_digit,
                                            sizeof(duk_unicode_re_ranges_digit) / sizeof(duk_uint16_t));
                ch = -1;
            } else if (x == 'D') {
                duk__emit_u16_direct_ranges(lex_ctx,
                                            gen_range,
                                            userdata,
                                            duk_unicode_re_ranges_not_digit,
                                            sizeof(duk_unicode_re_ranges_not_digit) / sizeof(duk_uint16_t));
                ch = -1;
            } else if (x == 's') {
                duk__emit_u16_direct_ranges(lex_ctx,
                                            gen_range,
                                            userdata,
                                            duk_unicode_re_ranges_white,
                                            sizeof(duk_unicode_re_ranges_white) / sizeof(duk_uint16_t));
                ch = -1;
            } else if (x == 'S') {
                duk__emit_u16_direct_ranges(lex_ctx,
                                            gen_range,
                                            userdata,
                                            duk_unicode_re_ranges_not_white,
                                            sizeof(duk_unicode_re_ranges_not_white) / sizeof(duk_uint16_t));
                ch = -1;
            } else if (x == 'w') {
                duk__emit_u16_direct_ranges(lex_ctx,
                                            gen_range,
                                            userdata,
                                            duk_unicode_re_ranges_wordchar,
                                            sizeof(duk_unicode_re_ranges_wordchar) / sizeof(duk_uint16_t));
                ch = -1;
            } else if (x == 'W') {
                duk__emit_u16_direct_ranges(lex_ctx,
                                            gen_range,
                                            userdata,
                                            duk_unicode_re_ranges_not_wordchar,
                                            sizeof(duk_unicode_re_ranges_not_wordchar) / sizeof(duk_uint16_t));
                ch = -1;
            } else if (DUK__ISDIGIT(x)) {
                /* DecimalEscape, only \0 is allowed, no leading zeroes are allowed */
                if (x == '0' && !DUK__ISDIGIT(DUK__L0())) {
                    ch = 0x0000;
                } else {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp escape");
                }
            } else if (!duk_unicode_is_identifier_part(x)
#if defined(DUK_USE_NONSTD_REGEXP_DOLLAR_ESCAPE)
                       || x == '$'
#endif
                      ) {
                /* IdentityEscape */
                ch = x;
            } else {
                DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp escape");
            }
        } else {
            /* character represents itself */
            ch = x;
        }
 
        /* ch is a literal character here or -1 if parsed entity was
         * an escape such as "\s".
         */
 
        if (ch < 0) {
            /* multi-character sets not allowed as part of ranges, see
             * E5 Section 15.10.2.15, abstract operation CharacterRange.
             */
            if (start >= 0) {
                if (dash) {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid range");
                } else {
                    gen_range(userdata, start, start, 0);
                    start = -1;
                    /* dash is already 0 */
                }
            }
        } else {
            if (start >= 0) {
                if (dash) {
                    if (start > ch) {
                        DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid range");
                    }
                    gen_range(userdata, start, ch, 0);
                    start = -1;
                    dash = 0;
                } else {
                    gen_range(userdata, start, start, 0);
                    start = ch;
                    /* dash is already 0 */
                }
            } else {
                start = ch;
            }
        }
    }
 
    return;
}

References DUK__ADVANCECHARS, duk__decode_hexesc_from_window(), duk__decode_uniesc_from_window(), duk__emit_u16_direct_ranges(), DUK__ISDIGIT, DUK__L0, DUK_ASSERT, DUK_DD, DUK_DDPRINT, DUK_ERROR_SYNTAX, duk_unicode_is_identifier_part(), duk_unicode_re_ranges_digit, duk_unicode_re_ranges_not_digit, duk_unicode_re_ranges_not_white, duk_unicode_re_ranges_not_wordchar, duk_unicode_re_ranges_white, duk_unicode_re_ranges_wordchar, DUK_UNREF, DUK_USE_NONSTD_REGEXP_DOLLAR_ESCAPE, and duk_lexer_ctx::thr.

duk_lexer_parse_re_token()

DUK_INTERNAL void duk_lexer_parse_re_token	(	duk_lexer_ctx *	lex_ctx,
		duk_re_token *	out_token )

Definition at line 1534 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                            {
    duk_small_int_t advtok = 0;  /* init is unnecessary but suppresses "may be used uninitialized" warnings */
    duk_codepoint_t x, y;
 
    if (++lex_ctx->token_count >= lex_ctx->token_limit) {
        DUK_ERROR_RANGE(lex_ctx->thr, "token limit");
        return;  /* unreachable */
    }
 
    DUK_MEMZERO(out_token, sizeof(*out_token));
 
    x = DUK__L0();
    y = DUK__L1();
 
    DUK_DDD(DUK_DDDPRINT("parsing regexp token, L0=%ld, L1=%ld", (long) x, (long) y));
 
    switch (x) {
    case '|': {
        advtok = DUK__ADVTOK(1, DUK_RETOK_DISJUNCTION);
        break;
    }
    case '^': {
        advtok = DUK__ADVTOK(1, DUK_RETOK_ASSERT_START);
        break;
    }
    case '$': {
        advtok = DUK__ADVTOK(1, DUK_RETOK_ASSERT_END);
        break;
    }
    case '?': {
        out_token->qmin = 0;
        out_token->qmax = 1;
        if (y == '?') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_QUANTIFIER);
            out_token->greedy = 0;
        } else {
            advtok = DUK__ADVTOK(1, DUK_RETOK_QUANTIFIER);
            out_token->greedy = 1;
        }
        break;
    }
    case '*': {
        out_token->qmin = 0;
        out_token->qmax = DUK_RE_QUANTIFIER_INFINITE;
        if (y == '?') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_QUANTIFIER);
            out_token->greedy = 0;
        } else {
            advtok = DUK__ADVTOK(1, DUK_RETOK_QUANTIFIER);
            out_token->greedy = 1;
        }
        break;
    }
    case '+': {
        out_token->qmin = 1;
        out_token->qmax = DUK_RE_QUANTIFIER_INFINITE;
        if (y == '?') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_QUANTIFIER);
            out_token->greedy = 0;
        } else {
            advtok = DUK__ADVTOK(1, DUK_RETOK_QUANTIFIER);
            out_token->greedy = 1;
        }
        break;
    }
    case '{': {
        /* Production allows 'DecimalDigits', including leading zeroes */
        duk_uint_fast32_t val1 = 0;
        duk_uint_fast32_t val2 = DUK_RE_QUANTIFIER_INFINITE;
        duk_small_int_t digits = 0;
#if defined(DUK_USE_ES6_REGEXP_BRACES)
        duk_lexer_point lex_pt;
#endif
 
#if defined(DUK_USE_ES6_REGEXP_BRACES)
        /* Store lexer position, restoring if quantifier is invalid. */
        DUK_LEXER_GETPOINT(lex_ctx, &lex_pt);
#endif
 
        for (;;) {
            DUK__ADVANCECHARS(lex_ctx, 1);  /* eat '{' on entry */
            x = DUK__L0();
            if (DUK__ISDIGIT(x)) {
                digits++;
                val1 = val1 * 10 + (duk_uint_fast32_t) duk__hexval(lex_ctx, x);
            } else if (x == ',') {
                if (digits > DUK__MAX_RE_QUANT_DIGITS) {
                    goto invalid_quantifier;
                }
                if (val2 != DUK_RE_QUANTIFIER_INFINITE) {
                    goto invalid_quantifier;
                }
                if (DUK__L1() == '}') {
                    /* form: { DecimalDigits , }, val1 = min count */
                    if (digits == 0) {
                        goto invalid_quantifier;
                    }
                    out_token->qmin = val1;
                    out_token->qmax = DUK_RE_QUANTIFIER_INFINITE;
                    DUK__ADVANCECHARS(lex_ctx, 2);
                    break;
                }
                val2 = val1;
                val1 = 0;
                digits = 0;  /* not strictly necessary because of lookahead '}' above */
            } else if (x == '}') {
                if (digits > DUK__MAX_RE_QUANT_DIGITS) {
                    goto invalid_quantifier;
                }
                if (digits == 0) {
                    goto invalid_quantifier;
                }
                if (val2 != DUK_RE_QUANTIFIER_INFINITE) {
                    /* val2 = min count, val1 = max count */
                    out_token->qmin = val2;
                    out_token->qmax = val1;
                } else {
                    /* val1 = count */
                    out_token->qmin = val1;
                    out_token->qmax = val1;
                }
                DUK__ADVANCECHARS(lex_ctx, 1);
                break;
            } else {
                goto invalid_quantifier;
            }
        }
        if (DUK__L0() == '?') {
            out_token->greedy = 0;
            DUK__ADVANCECHARS(lex_ctx, 1);
        } else {
            out_token->greedy = 1;
        }
        advtok = DUK__ADVTOK(0, DUK_RETOK_QUANTIFIER);
        break;
 invalid_quantifier:
#if defined(DUK_USE_ES6_REGEXP_BRACES)
        /* Failed to match the quantifier, restore lexer and parse
         * opening brace as a literal.
         */
        DUK_LEXER_SETPOINT(lex_ctx, &lex_pt);
        advtok = DUK__ADVTOK(1, DUK_RETOK_ATOM_CHAR);
        out_token->num = '{';
#else
        DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp quantifier");
#endif
        break;
    }
    case '.': {
        advtok = DUK__ADVTOK(1, DUK_RETOK_ATOM_PERIOD);
        break;
    }
    case '\\': {
        /* The E5.1 specification does not seem to allow IdentifierPart characters
         * to be used as identity escapes.  Unfortunately this includes '$', which
         * cannot be escaped as '\$'; it needs to be escaped e.g. as '\u0024'.
         * Many other implementations (including V8 and Rhino, for instance) do
         * accept '\$' as a valid identity escape, which is quite pragmatic.
         * See: test-regexp-identity-escape-dollar.js.
         */
 
        advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_CHAR);  /* default: char escape (two chars) */
        if (y == 'b') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ASSERT_WORD_BOUNDARY);
        } else if (y == 'B') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ASSERT_NOT_WORD_BOUNDARY);
        } else if (y == 'f') {
            out_token->num = 0x000c;
        } else if (y == 'n') {
            out_token->num = 0x000a;
        } else if (y == 't') {
            out_token->num = 0x0009;
        } else if (y == 'r') {
            out_token->num = 0x000d;
        } else if (y == 'v') {
            out_token->num = 0x000b;
        } else if (y == 'c') {
            x = DUK__L2();
            if ((x >= 'a' && x <= 'z') ||
                (x >= 'A' && x <= 'Z')) {
                out_token->num = (x % 32);
                advtok = DUK__ADVTOK(3, DUK_RETOK_ATOM_CHAR);
            } else {
                DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp escape");
            }
        } else if (y == 'x') {
            out_token->num = duk__decode_hexesc_from_window(lex_ctx, 2);
            advtok = DUK__ADVTOK(4, DUK_RETOK_ATOM_CHAR);
        } else if (y == 'u') {
            out_token->num = duk__decode_uniesc_from_window(lex_ctx, 2);
            advtok = DUK__ADVTOK(6, DUK_RETOK_ATOM_CHAR);
        } else if (y == 'd') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_DIGIT);
        } else if (y == 'D') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_NOT_DIGIT);
        } else if (y == 's') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_WHITE);
        } else if (y == 'S') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_NOT_WHITE);
        } else if (y == 'w') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_WORD_CHAR);
        } else if (y == 'W') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_NOT_WORD_CHAR);
        } else if (DUK__ISDIGIT(y)) {
            /* E5 Section 15.10.2.11 */
            if (y == '0') {
                if (DUK__ISDIGIT(DUK__L2())) {
                    DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp escape");
                }
                out_token->num = 0x0000;
                advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_CHAR);
            } else {
                /* XXX: shared parsing? */
                duk_uint_fast32_t val = 0;
                duk_small_int_t i;
                for (i = 0; ; i++) {
                    if (i >= DUK__MAX_RE_DECESC_DIGITS) {
                        DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp escape");
                    }
                    DUK__ADVANCECHARS(lex_ctx, 1);  /* eat backslash on entry */
                    x = DUK__L0();
                    if (!DUK__ISDIGIT(x)) {
                        break;
                    }
                    val = val * 10 + (duk_uint_fast32_t) duk__hexval(lex_ctx, x);
                }
                /* DUK__L0() cannot be a digit, because the loop doesn't terminate if it is */
                advtok = DUK__ADVTOK(0, DUK_RETOK_ATOM_BACKREFERENCE);
                out_token->num = val;
            }
        } else if ((y >= 0 && !duk_unicode_is_identifier_part(y)) ||
#if defined(DUK_USE_NONSTD_REGEXP_DOLLAR_ESCAPE)
                   y == '$' ||
#endif
                   y == DUK_UNICODE_CP_ZWNJ ||
                   y == DUK_UNICODE_CP_ZWJ) {
            /* IdentityEscape, with dollar added as a valid additional
             * non-standard escape (see test-regexp-identity-escape-dollar.js).
             * Careful not to match end-of-buffer (<0) here.
             */
            out_token->num = y;
        } else {
            DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp escape");
        }
        break;
    }
    case '(': {
        /* XXX: naming is inconsistent: ATOM_END_GROUP ends an ASSERT_START_LOOKAHEAD */
 
        if (y == '?') {
            if (DUK__L2() == '=') {
                /* (?= */
                advtok = DUK__ADVTOK(3, DUK_RETOK_ASSERT_START_POS_LOOKAHEAD);
            } else if (DUK__L2() == '!') {
                /* (?! */
                advtok = DUK__ADVTOK(3, DUK_RETOK_ASSERT_START_NEG_LOOKAHEAD);
            } else if (DUK__L2() == ':') {
                /* (?: */
                advtok = DUK__ADVTOK(3, DUK_RETOK_ATOM_START_NONCAPTURE_GROUP);
            }
        } else {
            /* ( */
            advtok = DUK__ADVTOK(1, DUK_RETOK_ATOM_START_CAPTURE_GROUP);
        }
        break;
    }
    case ')': {
        advtok = DUK__ADVTOK(1, DUK_RETOK_ATOM_END_GROUP);
        break;
    }
    case '[': {
        /*
         *  To avoid creating a heavy intermediate value for the list of ranges,
         *  only the start token ('[' or '[^') is parsed here.  The regexp
         *  compiler parses the ranges itself.
         */
        advtok = DUK__ADVTOK(1, DUK_RETOK_ATOM_START_CHARCLASS);
        if (y == '^') {
            advtok = DUK__ADVTOK(2, DUK_RETOK_ATOM_START_CHARCLASS_INVERTED);
        }
        break;
    }
#if !defined(DUK_USE_ES6_REGEXP_BRACES)
    case '}':
#endif
    case ']': {
        /* Although these could be parsed as PatternCharacters unambiguously (here),
         * E5 Section 15.10.1 grammar explicitly forbids these as PatternCharacters.
         */
        DUK_ERROR_SYNTAX(lex_ctx->thr, "invalid regexp character");
        break;
    }
    case -1: {
        /* EOF */
        advtok = DUK__ADVTOK(0, DUK_TOK_EOF);
        break;
    }
    default: {
        /* PatternCharacter, all excluded characters are matched by cases above */
        advtok = DUK__ADVTOK(1, DUK_RETOK_ATOM_CHAR);
        out_token->num = x;
        break;
    }
    }
 
    /*
     *  Shared exit path
     */
 
    DUK__ADVANCEBYTES(lex_ctx, advtok >> 8);
    out_token->t = advtok & 0xff;
}

References DUK__ADVANCEBYTES, DUK__ADVANCECHARS, DUK__ADVTOK, duk__decode_hexesc_from_window(), duk__decode_uniesc_from_window(), duk__hexval(), DUK__ISDIGIT, DUK__L0, DUK__L1, DUK__L2, DUK__MAX_RE_DECESC_DIGITS, DUK__MAX_RE_QUANT_DIGITS, DUK_DDD, DUK_DDDPRINT, DUK_ERROR_RANGE, DUK_ERROR_SYNTAX, DUK_LEXER_GETPOINT, DUK_LEXER_SETPOINT, DUK_MEMZERO, DUK_RE_QUANTIFIER_INFINITE, DUK_RETOK_ASSERT_END, DUK_RETOK_ASSERT_NOT_WORD_BOUNDARY, DUK_RETOK_ASSERT_START, DUK_RETOK_ASSERT_START_NEG_LOOKAHEAD, DUK_RETOK_ASSERT_START_POS_LOOKAHEAD, DUK_RETOK_ASSERT_WORD_BOUNDARY, DUK_RETOK_ATOM_BACKREFERENCE, DUK_RETOK_ATOM_CHAR, DUK_RETOK_ATOM_DIGIT, DUK_RETOK_ATOM_END_GROUP, DUK_RETOK_ATOM_NOT_DIGIT, DUK_RETOK_ATOM_NOT_WHITE, DUK_RETOK_ATOM_NOT_WORD_CHAR, DUK_RETOK_ATOM_PERIOD, DUK_RETOK_ATOM_START_CAPTURE_GROUP, DUK_RETOK_ATOM_START_CHARCLASS, DUK_RETOK_ATOM_START_CHARCLASS_INVERTED, DUK_RETOK_ATOM_START_NONCAPTURE_GROUP, DUK_RETOK_ATOM_WHITE, DUK_RETOK_ATOM_WORD_CHAR, DUK_RETOK_DISJUNCTION, DUK_RETOK_QUANTIFIER, DUK_TOK_EOF, DUK_UNICODE_CP_ZWJ, DUK_UNICODE_CP_ZWNJ, duk_unicode_is_identifier_part(), DUK_USE_NONSTD_REGEXP_DOLLAR_ESCAPE, duk_re_token::greedy, duk_re_token::num, duk_re_token::qmax, duk_re_token::qmin, duk_re_token::t, duk_lexer_ctx::thr, duk_lexer_ctx::token_count, and duk_lexer_ctx::token_limit.

duk_lexer_setpoint()

DUK_INTERNAL void duk_lexer_setpoint	(	duk_lexer_ctx *	lex_ctx,
		duk_lexer_point *	pt )

Definition at line 583 of file duktape-1.5.2/src-separate/duk_lexer.c.

                                                                                  {
    DUK_ASSERT_DISABLE(pt->offset >= 0);  /* unsigned */
    DUK_ASSERT(pt->line >= 1);
    lex_ctx->input_offset = pt->offset;
    lex_ctx->input_line = pt->line;
    duk__init_lexer_window(lex_ctx);
}

References duk__init_lexer_window(), DUK_ASSERT, DUK_ASSERT_DISABLE, duk_lexer_ctx::input_line, duk_lexer_ctx::input_offset, duk_lexer_point::line, and duk_lexer_point::offset.