/****************************************************************************
 * KONOHA COPYRIGHT, LICENSE NOTICE, AND DISCRIMER  
 * 
 * Copyright (c) 2005-2008, Kimio Kuramitsu <kimio at ynu.ac.jp>
 *           (c) 2008-      Konoha Software Foundation  
 * All rights reserved.
 * 
 * You may choose one of the following two licenses when you use konoha. 
 * See www.konohaware.org/license.html for further information.
 * 
 * (1) GNU General Public License 2.0      (with    KONOHA_UNDER_GPL2)
 * (2) Konoha Software Foundation License 1.0
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *  
 ****************************************************************************/

/* ************************************************************************ */

#include"commons.h"

/* ************************************************************************ */

#ifdef __cplusplus 
extern "C" {
#endif

/* ======================================================================== */
/* [data] */ 

#define PSTACK_MAXSIZ 32

typedef struct {  /* @data */
	struct knh_Token_t *tk;
	int indent;
} pstack_t;

static knh_bool_t knh_Bytes_isTripleQuote(Bytes *b, knh_int_t ch);
static pstack_t *knh_pstack_back(Ctx *ctx, pstack_t *ps, pstack_t *psp, int indent);
static Token *new_Token__buffer(Ctx *ctx, knh_token_t tt, knh_wbuf_t tbuf, InputStream *in);
static void knh_Token_padd(Ctx *ctx, Token *b, Token *tk);
#define knh_Token_add_whitespace  knh_Token_add_space
static void knh_Token_add_space(Ctx *ctx, Token *b, knh_wbuf_t tbuf, InputStream *in);

static int knh_InputStream_getc__urlencoded(Ctx *ctx, InputStream *in);

/* ======================================================================== */
/* [parse] */
	
void knh_Token_parse(Ctx *ctx, Token *tk, InputStream *in)
{
	knh_wbuf_t tbuf = knh_Context_wbuf(ctx);
	int ch, prev=0, indent = 0;
	pstack_t ps[PSTACK_MAXSIZ], *psp = ps;
	psp->tk = tk;
	psp->indent = 0;
	
	INDENT_PART:;
	indent = 0;
	while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
		if(ch == '\t') {
			indent+=3;
		}
		else if(ch == ' ') {
			indent++;
		}
		else if(ch == '/' || ch == '\n' || ch == '\r' || ch == EOF) {
			goto MAIN_PART_INLOOP;
		}
		break;
	}
	if(indent < psp->indent) {
		DEBUG3("deindent %d -> %d", indent, psp->indent);
		psp = knh_pstack_back(ctx, ps, psp, indent);
	}
	goto MAIN_PART_INLOOP;
	
	MAIN_PART:;
	while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
		MAIN_PART_INLOOP:;
		
		switch(ch) {
		case '\n':
			knh_Token_add_whitespace(ctx, psp->tk, tbuf, in);
		goto INDENT_PART;

		case '\t': case ' ': case '\v': case '\r':
			knh_Token_add_whitespace(ctx, psp->tk, tbuf, in);
			prev = ch;
		goto MAIN_PART;

		case '{':
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			if(SP(psp->tk)->tt == TT_PARENTHESIS && IS_NOTNULL(DP(psp->tk)->data)) {
				DBG2_P("@Probabry forget ')': indent=%d, psp->indent=%d", indent, psp->indent);
				knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_EFORGET_PARENTHESIS, "..) {");
				psp = knh_pstack_back(ctx, ps, psp, indent);
			}
		case '(': 	case '[':
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			if(psp+1 < ps+PSTACK_MAXSIZ) {
				Token *intk = new_Token(ctx, 0, DP(in)->fileid, DP(in)->line, knh_char_totoken(ch));
				knh_Token_padd(ctx, psp->tk, intk);
				psp++;
				psp->tk = intk;
				psp->indent = indent;
			}
			else {
				DBG2_P("pstack overflow!!");
				knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_EFORGET_PARENTHESIS, "), }, ]");
			}
			goto MAIN_PART;
			
		case '}':  
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			if(SP(psp->tk)->tt == TT_BRACE) {
				knh_Token_retokens(ctx, psp->tk);
				psp--;
				if(psp < ps) {
					knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_EMISMATCH, "{ ...}");
					psp++;
				}
			}
			else {
				knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_EMISMATCH, "{ ...}");
			}
			goto MAIN_PART;

		case ')': 
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			if(SP(psp->tk)->tt == TT_PARENTHESIS) {
				knh_Token_retokens(ctx, psp->tk);
				psp--;
				KNH_ASSERT(ps <= psp);
			}
			else {
				knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_EMISMATCH, "( ...)");
			}
			goto MAIN_PART;

		case ']': 
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			if(SP(psp->tk)->tt == TT_BRANCET) {
				knh_Token_retokens(ctx, psp->tk);
				psp--;
				KNH_ASSERT(ps <= psp);
			}
			else {
				knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_EMISMATCH, "[ ...]");
			}
			goto MAIN_PART;
		
		case '"': case '\'': case '`' :
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			prev = ch;
			goto QUOTED_PART;

		case ',': case ';':
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			knh_Token_padd(ctx, psp->tk, new_Token(ctx, 0, DP(in)->fileid, DP(in)->line, knh_char_totoken(ch)));
			break;

		case '$': 
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			prev = ch;
			goto QNAME_PART;
				
		case '@':
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			ch = knh_InputStream_getc(ctx, in); 
			if(ch == '"' || ch == '\'' || ch == '`') {
				prev = ch;
				goto RAWSTR_PART;
			}
			prev = '@';
			knh_Bytes_putc(ctx, tbuf.ba, ch);
			goto QNAME_PART;

		case '%': 
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			ch = knh_InputStream_getc(ctx, in); 
			if(isalnum(ch)) {
				prev = '%';
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				goto QNAME_PART;
			}
			else {
				knh_Bytes_putc(ctx, tbuf.ba, '%');
				knh_Token_add_space(ctx, psp->tk, tbuf, in);
				goto MAIN_PART_INLOOP;
			}
		
		case '+': case '-': case '*': case '=':
		case '&': case '|': case '<':
		case '>': case '^': case '!': case '~':
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			knh_Bytes_putc(ctx, tbuf.ba, ch);
			prev = ch;
			goto OP_PART;
		case '?':
			if(!islower(prev)) {
				knh_Token_add_space(ctx, psp->tk, tbuf, in);
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				prev = ch;
				goto OP_PART;
			}
			else {
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				knh_Token_add_space(ctx, psp->tk, tbuf, in);
				break;
			}
			
		case '/':
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			ch = knh_InputStream_getc(ctx, in); 
			if(ch == '*') {
				goto BLOCK_COMMENT;
			}else if(ch == '/') {
				goto LINE_COMMENT;
			}
			knh_Bytes_putc(ctx, tbuf.ba, '/');
			prev = '/';
			goto OP_PART_INLOOP;

		case '#':
			knh_Token_add_space(ctx, psp->tk, tbuf, in);
			goto LINE_COMMENT;

		case '.':
			ch = knh_InputStream_getc(ctx, in); 
			if(ch == '.') { /* .. */
				knh_Token_add_space(ctx, psp->tk, tbuf, in);
				knh_Bytes_putc(ctx, tbuf.ba, '.');
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				prev = '.';
				goto OP_PART;
			}
			else if(ch == '*') {
				knh_Bytes_putc(ctx, tbuf.ba, '.');
				knh_Bytes_putc(ctx, tbuf.ba, '*');
				knh_Token_add_space(ctx, psp->tk, tbuf, in);
				prev = ' ';
				break;
			}
			else { /* a.b */
				knh_Bytes_putc(ctx, tbuf.ba, '.');
				goto MAIN_PART_INLOOP;
			}
			break; 
		
		case ':':
			ch = knh_InputStream_getc(ctx, in);
			if(knh_wbuf_size(tbuf) == 0) {
				if(ch == ' ' || ch == '\n' || ch == '\t' || ch == '\r' || ch == ';') {
					knh_Bytes_putc(ctx, tbuf.ba, ':');
					knh_Token_add_space(ctx, psp->tk, tbuf, in);
					break;
				}
				knh_Bytes_putc(ctx, tbuf.ba, ':');
				goto URN_PART_INLOOP;
			}
			if(ch == '/') {
				knh_Bytes_putc(ctx, tbuf.ba, ':');
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				goto URN_PART;
			}
			else if(ch == ' ' || ch == '\n' || ch == '\t' || ch == '\r' || ch == ';') {
				knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_LABEL, tbuf, in));
				//knh_Bytes_putc(ctx, tbuf.ba, ':');
				goto MAIN_PART_INLOOP;
			}
			else {
				knh_Bytes_putc(ctx, tbuf.ba, ':');
				knh_Bytes_putc(ctx, tbuf.ba, ch);
			}
			break;
		
		case '1': case '2': case '3': case '4': case '5':
		case '6': case '7': case '8': case '9': case '0':
			if(knh_wbuf_size(tbuf) == 0) {
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				goto NUM_PART;
			}
			else {
				knh_Bytes_putc(ctx, tbuf.ba, ch);
			}

		case EOF :
			break;

		default:
			if(ch > 127) {
				knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_WASCII, NULL);
			}
			knh_Bytes_putc(ctx, tbuf.ba, ch);
		} /* switch */
		prev = ch;
	}/* while */
	knh_Token_add_space(ctx, psp->tk, tbuf, in);
	return ;

	NUM_PART:;  /* 0x0000 */
	{
		int unit = 0;
		//DEBUG3("=> NUM_PART");
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
			NUM_PART_INLOOP:;
			switch(ch) {
				case '0': case '1': case '2': case '3': case '4':
				case '5': case '6': case '7': case '8': case '9':
				case '_':
					knh_Bytes_putc(ctx, tbuf.ba, ch);
				break;
				case '.':
					ch = knh_InputStream_getc(ctx, in);
					if(ch == '.') {
						knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_NUM, tbuf, in));
						knh_Bytes_putc(ctx, tbuf.ba, '.');
						goto OP_PART_INLOOP;
					}
					else {
						knh_Bytes_putc(ctx, tbuf.ba, '.');
						goto NUM_PART_INLOOP;
					}
				break;
				
				case '[':
					unit++;
					knh_Bytes_putc(ctx, tbuf.ba, ch);
				break;
					
				case ']':   /* 1.0[km/h] */
					unit--;
					if(unit < 0) {
						knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_NUM, tbuf, in));
						goto MAIN_PART_INLOOP;
					}
					else if(unit == 0) {
						knh_Bytes_putc(ctx, tbuf.ba, ch);
						knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_NUM, tbuf, in));
						goto MAIN_PART;
					}
				break;
					
				default:
					if(unit > 0 || isalnum(ch) || ch > 127) {  /* 1.0a */
						knh_Bytes_putc(ctx, tbuf.ba, ch);
					}
					else {
						knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_NUM, tbuf, in));
						goto MAIN_PART_INLOOP;
					}
				/*break*/
			}
		}
		knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_NUM, tbuf, in));
		//knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_WEOF, NULL);
	}
	goto MAIN_PART_INLOOP; /* EOF */

	QNAME_PART:;
	KNH_ASSERT(prev == '@' || prev == '$' || prev == '%');
	{
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
			if(isalnum(ch) || ch == '_' || ch == ':') {
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				continue;
			}
			if(ch == '.' && prev != '%') {
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				continue;
			}
			if(ch == '*' && prev == '$') {
				knh_Bytes_putc(ctx, tbuf.ba, ch);
				continue;
			}
			break;
		}
		knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, knh_char_totoken(prev), tbuf, in));
	}
	goto MAIN_PART_INLOOP; /* EOF */
	
	OP_PART:;
	{
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
			OP_PART_INLOOP:;
			switch(ch) {
				case '-': case '|': case '&':
					if(prev == '=' || prev == '>' || prev == '<') {
						knh_Token_add_space(ctx, psp->tk, tbuf, in);
					}
					knh_Bytes_putc(ctx, tbuf.ba, ch);
					prev = ch;
					break;
				case '=': case '+': case '<': case '>':
				case '.': case '?': case ':': 
					knh_Bytes_putc(ctx, tbuf.ba, ch);
					prev = ch;
					break;
				default:
					knh_Token_add_space(ctx, psp->tk, tbuf, in);
					goto MAIN_PART_INLOOP;
			}
		}
	}
	goto MAIN_PART_INLOOP; /* EOF */

	URN_PART:;
	{
		DEBUG3("=> URN_PART");
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
			URN_PART_INLOOP:;
			switch(ch) {
				case '\n': case '\r':
				case ' ':  case '\t':
				case ';':  
				case '(': case ')':
				case '{': case '}':
				case '[': case ']':
					knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_URN, tbuf, in));
					goto MAIN_PART_INLOOP;
				case '%':
					ch = knh_InputStream_getc__urlencoded(ctx, in);
					if(ch == EOF) {
						ch = '\n';
						knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_URN, tbuf, in));
						goto MAIN_PART_INLOOP;
					}
					knh_Bytes_putc(ctx, tbuf.ba, ch); 
				default:
					knh_Bytes_putc(ctx, tbuf.ba, ch);
			}
		}
		knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_URN, tbuf, in));
	}/*URN_PART*/
	return;

	RAWSTR_PART:;
	KNH_ASSERT(prev == '"' || prev == '\'' || prev == '`');
	{
		knh_Bytes_putc(ctx, tbuf.ba, prev);
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
			if(ch == prev) {
				knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_STR, tbuf, in));
				goto MAIN_PART;
			}
			knh_Bytes_putc(ctx, tbuf.ba, ch);
		}
		{
			char buf4[4];
			knh_snprintf(buf4, sizeof(buf4), "%c", prev);
			knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_WUNCLOSED, buf4);
			knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_STR, tbuf, in));
			goto MAIN_PART_INLOOP;
		}
	}/*RAWSTR_PART*/
	return;

	QUOTED_PART:;
	KNH_ASSERT(prev == '"' || prev == '\'' || prev == '`');
	{
		knh_Bytes_putc(ctx, tbuf.ba, prev);
		ch = knh_InputStream_getc(ctx, in); 
		if(ch == prev) {
			ch = knh_InputStream_getc(ctx, in); 
			if(prev == ch) { /* triple quote */
				goto TRIPLE_QUOTED_PART;		
			}else {
				knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_STR, tbuf, in));
				goto MAIN_PART_INLOOP;
			}
		}
		else {
			goto QUOTED_PART_INLOOP;
		}
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
			QUOTED_PART_INLOOP:;
			if(ch == '\\') {
				ch = knh_InputStream_getc(ctx, in);
				switch(ch) {
				case 'n': 
					knh_Bytes_putc(ctx, tbuf.ba, '\n');
					break;
				case 't': 
					knh_Bytes_putc(ctx, tbuf.ba, '\t');
					break;
				case '"': case '\'': case '\\': 
					knh_Bytes_putc(ctx, tbuf.ba, ch);
					break;
				case EOF:
					
				default:
					knh_Bytes_putc(ctx, tbuf.ba, '\\');
					knh_Bytes_putc(ctx, tbuf.ba, ch);
				}
				continue;
			}
			if(ch == '\n' || ch == '\r' || ch == EOF) {
				char buf4[4];
				knh_snprintf(buf4, sizeof(buf4), "%c", prev);
				knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_WUNCLOSED, buf4);
				knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_STR, tbuf, in));
				goto MAIN_PART_INLOOP;
			}
			if(ch == prev) {
				knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_STR, tbuf, in));
				goto MAIN_PART;
			}
			knh_Bytes_putc(ctx, tbuf.ba, ch);
		}
	}
	return; /* EOF */
	
	TRIPLE_QUOTED_PART:;
	KNH_ASSERT(prev == '"' || prev == '\'' || prev == '`');
	{
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) { 
			if(ch == '\\') {
				ch = knh_InputStream_getc(ctx, in);
				switch(ch) {
				case 'n': 
					knh_Bytes_putc(ctx, tbuf.ba, '\n');
					break;
				case 't': 
					knh_Bytes_putc(ctx, tbuf.ba, '\t');
					break;
				case '"': case '\'': case '\\': 
					knh_Bytes_putc(ctx, tbuf.ba, ch);
					break;
				case EOF:
					
				default:
					knh_Bytes_putc(ctx, tbuf.ba, '\\');
					knh_Bytes_putc(ctx, tbuf.ba, ch);
				}
				continue;
			}
			if(ch == '\n') {
				if(knh_Bytes_size(tbuf.ba) - tbuf.pos == 1) {
					DBG2_P("remove first linefeed at triplequote");
					continue;
				}
			}
			knh_Bytes_putc(ctx, tbuf.ba, ch);
			if(knh_Bytes_isTripleQuote(tbuf.ba, prev)) {
				knh_Bytes_unputc(tbuf.ba);
				knh_Bytes_unputc(tbuf.ba);
				knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_STR, tbuf, in));
				goto MAIN_PART;
			}
		}
		{
			char buf4[4];
			knh_snprintf(buf4, sizeof(buf4), "%c%c%c", prev, prev, prev);
			knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_WUNCLOSED, buf4);
			knh_Token_padd(ctx, psp->tk, new_Token__buffer(ctx, TT_STR, tbuf, in));
		}
	}
	return;
	
	/** COMMENT **/
	LINE_COMMENT:
	{
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
			if(ch == '\n') {
				goto MAIN_PART_INLOOP;
			}
		}
	}
	return ;

	BLOCK_COMMENT:
	{
		int nest = 1;
		while((ch = knh_InputStream_getc(ctx, in)) != EOF) {
			if(prev == '*' && ch == '/') {
				nest--;
				if(nest == 0) goto MAIN_PART;
			}else if(prev == '/' && ch == '*') {
				DBG2_P("nesting /* ")
				nest++;
			}
			prev = ch;
		}
		knh_perror(ctx, DP(in)->fileid, DP(in)->line, KMSG_WUNCLOSED, "/* ... */");
	}
	return ;			
}

/* ------------------------------------------------------------------------ */

String *new_String__NAME(Ctx *ctx, knh_bytes_t tname)
{
	knh_index_t idx = knh_DictIdx_index(ctx, DP(knh_rootSystem)->tfieldnDictIdx, tname);
	if(idx != -1) {
		return knh_DictIdx_get__fast(DP(knh_rootSystem)->tfieldnDictIdx, (knh_fieldn_t)idx);
	}
	DEBUG3("new_String__NAME('%s')", (char*)tname.buf);
	return new_String(ctx, tname, NULL);
}

/* ------------------------------------------------------------------------ */

#ifndef KONOHA_NAME_BUFSIZ
#define KONOHA_NAME_BUFSIZ  256
#endif

#ifndef KONOHA_NAME_MAXSIZ
#define KONOHA_NAME_MAXSIZ  80
#endif

Token *new_Token__name(Ctx *ctx, knh_flag_t flag, knh_fileid_t fileid, knh_line_t line, knh_bytes_t t)
{
	char name[KONOHA_NAME_BUFSIZ], *p = name;
	knh_token_t tt = TT_NAME;
	int i = 0;
	if(t.buf[0] == '.' && islower(t.buf[1])) {   /* .name   => name */
		flag |= KNH_FLAG_TKF_TOPDOT;
		t.buf++; t.len--;
	}
	if(isupper(t.buf[0])) { /* CLASS */
		tt = TT_TYPEN;
		goto CLASS_PART;
	}
	else if(islower(t.buf[0]) || t.buf[0] == '_'){
		tt = TT_NAME;
		goto NAME_PART;
	}
	else {
		knh_perror(ctx, fileid, line, KMSG_ETOKEN, (char*)t.buf);
		knh_snprintf(name, sizeof(name), "%s", (char*)t.buf);
		tt = TT_ERR;
		goto TOKEN_PART;
	}
	
	CLASS_PART:
	{
		int has_lower = 0;
		for(; i < t.len; i++) {
			if(p - name >= KONOHA_NAME_MAXSIZ) {
				knh_perror(ctx, fileid, line, KMSG_ELENGTH, (char*)t.buf);
				*p = 0;
				knh_perrata(ctx, fileid, line, (char*)t.buf, name);
				goto TOKEN_PART;
			}
			if(t.buf[i] == '.') {
				if(isupper(t.buf[i+1])) {
					tt = TT_CONSTN;
					*p = '.'; p++; i++;
					goto CONSTN_PART;
				}
				else if(islower(t.buf[i+1])){
					tt = TT_CMETHODN;
					*p = '.'; p++; i++;
					goto NAME_PART;
				}
			}
			if(t.buf[i] == '_') {
				if(has_lower) {
					knh_perror(ctx, fileid, line, KMSG_WCLASSN, (char*)t.buf);
					*p = 0;
					knh_perrata(ctx, fileid, line, (char*)t.buf, name);
					goto TOKEN_PART;
				}
				else {
					tt = TT_CONSTN;
					*p = t.buf[i]; p++; i++;
					goto CONSTN_PART;
				}
			}
			if(!isalnum(t.buf[i]) && t.buf[i] != ':') {
				knh_perror(ctx, fileid, line, KMSG_WCLASSN, (char*)t.buf);
				*p = 0;
				knh_perrata(ctx, fileid, line, (char*)t.buf, name);
				goto TOKEN_PART;
			}
			if(islower(t.buf[i])) has_lower = 1;
			*p = t.buf[i]; p++;
		}
		*p = 0;
		goto TOKEN_PART;
	}

	CONSTN_PART:
	{
		for(; i < t.len; i++) {
			if(p - name >= KONOHA_NAME_MAXSIZ) {
				knh_perror(ctx, fileid, line, KMSG_ELENGTH, (char*)t.buf);
				*p = 0;
				knh_perrata(ctx, fileid, line, (char*)t.buf, name);
				goto TOKEN_PART;
			}
			if(isupper(t.buf[i]) || isdigit(t.buf[i]) || t.buf[i] == '_') {
				*p = t.buf[i]; p++;
			}
			else if(t.buf[i] == '.' && islower(t.buf[i+1])) {
				tt = TT_NAME;
				*p = '.'; p++; i++;
				goto NAME_PART;
			}
			else {
				knh_perror(ctx, fileid, line, KMSG_WCONSTN, (char*)t.buf);
				*p = 0;
				knh_perrata(ctx, fileid, line, (char*)t.buf, name);
				goto TOKEN_PART;
			}
		}
		*p = 0;
		goto TOKEN_PART;
	}
	
	NAME_PART:
	{
		int upper = 0;
		for(; t.buf[i] == '_'; i++) {
			if(p - name >= KONOHA_NAME_MAXSIZ) {
				knh_perror(ctx, fileid, line, KMSG_ELENGTH, (char*)t.buf);
				*p = 0;
				knh_perrata(ctx, fileid, line, (char*)t.buf, name);
				goto TOKEN_PART;
			}
			*p = t.buf[i]; p++;
		}
		
		for(; i < t.len; i++) {
			if(p - name >= KONOHA_NAME_MAXSIZ) {
				knh_perror(ctx, fileid, line, KMSG_ELENGTH, (char*)t.buf);
				*p = 0;
				knh_perrata(ctx, fileid, line, (char*)t.buf, name);
				goto TOKEN_PART;
			}
			if(t.buf[i] == '_') {
				if(upper) {
					*p = '_'; p++;
					*p = '_'; p++;
				} 
				upper = 1;
				continue;
			}
			if(!isalnum(t.buf[i]) && t.buf[i] != ':') {
				if(t.buf[i] == '.' || tt != TT_CMETHODN) {
					*p = '.'; p++;
					continue;
				}
				knh_perror(ctx, fileid, line, KMSG_WNAME, (char*)t.buf);
				*p = 0;
				knh_perrata(ctx, fileid, line, (char*)t.buf, name);
				goto TOKEN_PART;
			}
			if(upper) {
				upper = 0;
				if(islower(t.buf[i])) {
					*p = t.buf[i] - ('a' - 'A'); p++;
				}
				else {
					*p = t.buf[i]; p++;
				}
			}
			else {
				*p = t.buf[i]; p++;
			}
		}
		*p = 0;
		goto TOKEN_PART;
	}
	
	TOKEN_PART: 
	if(tt == TT_NAME) {
		if(name[0] == 'v' && name[1] == 'o' && name[2] == 'i' && name[3] == 'd' && name[4] == 0) {
			tt = TT_TYPEN;
		}
		if(name[0] == 'a' && name[1] == 'n' && name[2] == 'y' && name[3] == 0) {
			name[0] = 'A';
			tt = TT_TYPEN;
		}
	}
	{
		Token *o = new_Token(ctx, flag, fileid, line, tt);
		DEBUG3("(%s) '%s' ==> '%s'", knh_token_tochar(tt), (char*)t.buf, name);
		KNH_SETv(ctx, DP(o)->data, new_String__NAME(ctx, B(name)));
		return o;
	}
}

/* ------------------------------------------------------------------------ */
static 
pstack_t *knh_pstack_back(Ctx *ctx, pstack_t *ps, pstack_t *psp, int indent)
{
	DEBUG3("@S indent=%d", indent);
	while(ps < psp) {
		psp--;
		if(SP(psp->tk)->tt != TT_BRACE) continue;
		DEBUG3("psp->indent=%d", psp->indent);
		if(psp->indent < indent) continue;
	}
	DEBUG3("@E psp->indent=%d", psp->indent);
	return psp;
}

/* ------------------------------------------------------------------------ */

static
knh_bool_t knh_Bytes_isTripleQuote(Bytes *o, knh_int_t ch)
{
	if(o->size > 2 && 
		o->buf[o->size-1] == ch && 
		o->buf[o->size-2] == ch &&
		o->buf[o->size-3] == ch) return 1;
	return 0;
}

/* ------------------------------------------------------------------------ */
/* [Token] */

static
Token *new_Token__buffer(Ctx *ctx, knh_token_t tt, knh_wbuf_t tbuf, InputStream *in)
{
	Token *o = new_Token(ctx, 0, DP(in)->fileid, DP(in)->line, tt);
	knh_bytes_t t = knh_wbuf_tobytes(tbuf);
	//DEBUG3("tt=%s, '%s'", knh_token_tochar(tt), t.buf);
	
	if(tt == TT_STR) {
		if(t.buf[0] == '`' && t.len > 1) {
			TODO3("add `quote`");
		}
		else if(t.buf[0] == '\'') {
			SP(o)->tt = TT_TSTR;
		}
		t = knh_bytes_last(t, 1);
	}
	if(IS_NULL(DP(in)->bconv)) {
		KNH_SETv(ctx, DP(o)->data, new_String(ctx, t, NULL));
	}
	else {
		KNH_SETv(ctx, DP(o)->data, new_String__bconv(ctx, t, DP(in)->bconv));
	}
	knh_wbuf_clear(tbuf);
	return o;
}


/* ------------------------------------------------------------------------ */

#define knh_token_isNested(tt)  (tt == TT_BRACE || tt == TT_PARENTHESIS || tt == TT_BRANCET)

/* ------------------------------------------------------------------------ */

static
Token *knh_Token_tokens_lastNULL(Token *o)
{
	KNH_ASSERT(knh_token_isNested(SP(o)->tt));
	if(IS_bArray(DP(o)->list)) {
		Token *tk = (Token*)knh_Array_last(DP(o)->list);
		KNH_ASSERT(IS_Token(tk));
		return tk;
	}
	else if(IS_Token(DP(o)->token)) {
		return DP(o)->token;
	}
	else {
		return NULL;
	}
}

/* ------------------------------------------------------------------------ */

static
void knh_Token_padd(Ctx *ctx, Token *o, Token *tk)
{
	Token *tkp = knh_Token_tokens_lastNULL(o);
	if(tkp != NULL) {
		DP(tkp)->tt_next = SP(tk)->tt;
	}
	knh_Token_tokens_add(ctx, o, tk);
}

/* ------------------------------------------------------------------------ */

void knh_Token_retokens(Ctx *ctx, Token *o) 
{
	Array *a = (Array*)DP(o)->data;
	if(IS_Array(a)) {
		int i;
		for(i = 0; i < knh_Array_size(a) - 1; i++) {
			Token *tk = (Token*)knh_Array_n(a, i);
			if(SP(tk)->tt == TT_TYPEN) {
				Token *tkn = (Token*)knh_Array_n(a, i+1);
				if(SP(tkn)->tt == TT_BRANCET && IS_NULL(DP(tkn)->data)) {
					DEBUG3("%s[]", knh_Token_tochar(tk));
					knh_Token_setArrayType(tk, 1);
					knh_Token_setNotNullType(tk, 1);
					DP(tk)->tt_next = DP(tkn)->tt_next;
					knh_Array_remove(ctx, a, i+1);
				}
			}
			if(SP(tk)->tt == TT_NAME && ISB(knh_Token_tobytes(tk), "byte")) {
				Token *tkn = (Token*)knh_Array_n(a, i+1);
				if(SP(tkn)->tt == TT_BRANCET && IS_NULL(DP(tkn)->data)) {
					DEBUG3("%s[]", knh_Token_tochar(tk));
					SP(tk)->tt = TT_TYPEN;
					KNH_SETv(ctx, DP(tk)->data, knh_tClass[CLASS_Bytes].sname);
					knh_Token_setNotNullType(tk, 1);
					if(i>0) {
						Token *tkp = (Token*)knh_Array_n(a, i-1);
						DP(tkp)->tt_next = TT_TYPEN;
					}
					DP(tk)->tt_next = DP(tkn)->tt_next;
					knh_Array_remove(ctx, a, i+1);
				}
			}
		}
	}
}

/* ------------------------------------------------------------------------ */

#define knh_Token_add_whitespace  knh_Token_add_space

static
void knh_Token_add_space(Ctx *ctx, Token *o, knh_wbuf_t tbuf, InputStream *in)
{
	knh_bytes_t t = knh_wbuf_tobytes(tbuf);
	//DEBUG3("t.len=%d, '%s'", (int)t.len, t.buf);
	
	if(t.len == 0) {
		return;
	}
	else {
//		if(t.buf[0] == '.' && t.len == 1) {
//			TODO3(". Token");
//			knh_wbuf_clear(tbuf);
//			return;
//		}
		Token *tkp = knh_Token_tokens_lastNULL(o);
		if(tkp != NULL) {
			if(SP(tkp)->tt == TT_TYPEN) {
				if(t.buf[0] == '.' && t.buf[1] == '.' && t.buf[2] == '\0') {
					knh_Token_setIteratorType(tkp, 1);
					knh_wbuf_clear(tbuf);
					return;
				}
				if(t.buf[0] == '!' && t.buf[1] == '!' && t.buf[2] == '\0') {
					knh_Token_setExceptionType(tkp, 1);
					knh_wbuf_clear(tbuf);
					return;
				}
				if(t.buf[0] == '!' && t.buf[2] == '\0') {
					knh_Token_setNotNullType(tkp, 1);
					knh_wbuf_clear(tbuf);
					return;
				}
			}
		}
		knh_Token_padd(ctx, o, new_Token__parse(ctx, 0, DP(in)->fileid, DP(in)->line, t, DP(in)->bconv));
		knh_wbuf_clear(tbuf);
	}
}

/* ------------------------------------------------------------------------ */

knh_token_t knh_char_totoken(int ch)
{
	switch(ch) {
		case '{': case '}': return TT_BRACE;
		case '(': case ')': return TT_PARENTHESIS;
		case '[': case ']': return TT_BRANCET;
		case '@': return TT_METAN;
		case '$': return TT_PROPN;
		case '%': return TT_MT;
		case ';': return TT_SEMICOLON;
		case ',': return TT_COMMA;
	}
	KNH_ASSERT(ch == -1);
	return TT_ERR;
}

/* ------------------------------------------------------------------------ */

static 
int knh_InputStream_getc__urlencoded(Ctx *ctx, InputStream *in)
{
	int num, ch = knh_InputStream_getc(ctx, in);
	if(isdigit(ch)) {
		num = (ch - '0') * 16;
	}else if('a' <= ch && ch <= 'f') {
		num = (ch - 'a' + 10) * 16;
	}else if('A' <= ch && ch <= 'F') {
		num = (ch - 'A' + 10) * 16;
	}else {
		if(ch == '\n') return EOF;
		return ch;
	}
	ch = knh_InputStream_getc(ctx, in);
	if(isdigit(ch)) {
		num += (ch - '0');
	}else if('a' <= ch && ch <= 'f') {
		num += (ch - 'a' + 10);
	}else if('A' <= ch && ch <= 'F') {
		num += (ch - 'A' + 10);
	}else {
		if(ch == '\n') return EOF;
		return ch;
	}
	DEBUG3("num=%d, '%c'", num, num);
	return num;
}

/* ======================================================================== */

#ifdef __cplusplus
}
#endif
