Lines Matching refs:token
54 struct token **tokenlist;
55 struct token *token; member
185 const char *show_token(const struct token *token) in show_token() argument
189 if (!token) in show_token()
191 switch (token_type(token)) { in show_token()
199 return show_ident(token->ident); in show_token()
202 return token->number; in show_token()
205 return show_special(token->special); in show_token()
208 return show_char(token->string->data, in show_token()
209 token->string->length - 1, 0, '\''); in show_token()
211 return show_char(token->embedded, in show_token()
212 token_type(token) - TOKEN_CHAR, 0, '\''); in show_token()
214 return show_char(token->string->data, in show_token()
215 token->string->length - 1, 'L', '\''); in show_token()
217 return show_char(token->embedded, in show_token()
218 token_type(token) - TOKEN_WIDE_CHAR, 'L', '\''); in show_token()
220 return show_char(token->string->data, in show_token()
221 token->string->length - 1, 0, '"'); in show_token()
223 return show_char(token->string->data, in show_token()
224 token->string->length - 1, 'L', '"'); in show_token()
227 sprintf(buffer, "<beginning of '%s'>", stream_name(token->pos.stream)); in show_token()
231 sprintf(buffer, "<end of '%s'>", stream_name(token->pos.stream)); in show_token()
243 sprintf(buffer, "unhandled token type '%d' ", token_type(token)); in show_token()
248 const char *quote_token(const struct token *token) in quote_token() argument
252 switch (token_type(token)) { in quote_token()
257 return show_ident(token->ident); in quote_token()
260 return token->number; in quote_token()
263 return show_special(token->special); in quote_token()
266 return quote_char(token->string->data, in quote_token()
267 token->string->length - 1, 0, '\''); in quote_token()
269 return quote_char(token->embedded, in quote_token()
270 token_type(token) - TOKEN_CHAR, 0, '\''); in quote_token()
272 return quote_char(token->string->data, in quote_token()
273 token->string->length - 1, 'L', '\''); in quote_token()
275 return quote_char(token->embedded, in quote_token()
276 token_type(token) - TOKEN_WIDE_CHAR, 'L', '\''); in quote_token()
278 return quote_char(token->string->data, in quote_token()
279 token->string->length - 1, 0, '"'); in quote_token()
281 return quote_char(token->string->data, in quote_token()
282 token->string->length - 1, 'L', '"'); in quote_token()
284 sprintf(buffer, "unhandled token type '%d' ", token_type(token)); in quote_token()
334 static struct token * alloc_token(stream_t *stream) in alloc_token()
336 struct token *token = __alloc_token(0); in alloc_token() local
337 token->pos = stream_pos(stream); in alloc_token()
338 return token; in alloc_token()
449 struct token eof_token_entry;
451 static struct token *mark_eof(stream_t *stream) in mark_eof()
453 struct token *end; in mark_eof()
471 struct token *token = stream->token; in add_token() local
473 stream->token = NULL; in add_token()
474 token->next = NULL; in add_token()
475 *stream->tokenlist = token; in add_token()
476 stream->tokenlist = &token->next; in add_token()
481 stream->newline |= stream->token->pos.newline; in drop_token()
482 stream->whitespace |= stream->token->pos.whitespace; in drop_token()
483 stream->token = NULL; in drop_token()
538 struct token *token; in get_one_number() local
568 token = stream->token; in get_one_number()
569 token_type(token) = TOKEN_NUMBER; in get_one_number()
570 token->number = xmemdup(buffer, p - buffer); in get_one_number()
580 struct token *token = stream->token; in eat_string() local
625 token_type(token) = type + len; in eat_string()
627 memcpy(token->embedded, buffer, 4); in eat_string()
629 token_type(token) = type; in eat_string()
634 token->string = string; in eat_string()
638 token = stream->token; in eat_string()
742 struct token *token; in get_one_special() local
784 token = stream->token; in get_one_special()
785 token_type(token) = TOKEN_SPECIAL; in get_one_special()
786 token->special = value; in get_one_special()
900 struct token *built_in_token(int stream, struct ident *ident) in built_in_token()
902 struct token *token; in built_in_token() local
904 token = __alloc_token(0); in built_in_token()
905 token->pos.stream = stream; in built_in_token()
906 token_type(token) = TOKEN_IDENT; in built_in_token()
907 token->ident = ident; in built_in_token()
908 return token; in built_in_token()
913 struct token *token; in get_one_identifier() local
946 token = stream->token; in get_one_identifier()
947 token_type(token) = TOKEN_IDENT; in get_one_identifier()
948 token->ident = ident; in get_one_identifier()
963 static struct token *setup_stream(stream_t *stream, int idx, int fd, in setup_stream()
966 struct token *begin; in setup_stream()
974 stream->token = NULL; in setup_stream()
986 static struct token *tokenize_stream(stream_t *stream) in tokenize_stream()
991 struct token *token = alloc_token(stream); in tokenize_stream() local
992 stream->token = token; in tokenize_stream()
1004 struct token * tokenize_buffer(void *buffer, unsigned long size, struct token **endtoken) in tokenize_buffer()
1007 struct token *begin; in tokenize_buffer()
1014 struct token * tokenize(const char *name, int fd, struct token *endtoken, const char **next_path) in tokenize()
1016 struct token *begin, *end; in tokenize()