Avoid re-expanding a macro name that has once been rejected from expansion.

The specification of the preprocessor in C99 says that when we see a
macro name that we are already expanding that we refuse to expand it
now, (which we've done for a while), but also that we refuse to ever
expand it later if seen in other contexts at which it would be
legitimate to expand.

We add a test case for that here, and fix it to work. The fix takes
advantage of a new token_t value for tokens and argument words along
with the recently added IDENTIFIER_FINALIZED token type which
instructs the parser to not even look for another expansion.

glcpp-lex.l

@@ -114,12 +114,14 @@ TOKEN		[^[:space:](),]+
 <ST_DEFINE_PARAMETER>{HSPACE}+
 
 <ST_DEFINE_VALUE>{TOKEN} {
-	yylval.str = xtalloc_strdup (yyextra, yytext);
+	yylval.token.type = TOKEN;
+	yylval.token.value = xtalloc_strdup (yyextra, yytext);
 	return TOKEN;
 }
 
 <ST_DEFINE_VALUE>[(),] {
-	yylval.str = xtalloc_strdup (yyextra, yytext);
+	yylval.token.type = TOKEN;
+	yylval.token.value = xtalloc_strdup (yyextra, yytext);
 	return TOKEN;
 }
 
@@ -147,6 +149,9 @@ TOKEN		[^[:space:](),]+
 		case TOKEN_CLASS_IDENTIFIER:
 			return IDENTIFIER;
 		break;
+		case TOKEN_CLASS_IDENTIFIER_FINALIZED:
+			return IDENTIFIER_FINALIZED;
+		break;
 		case TOKEN_CLASS_FUNC_MACRO:
 			return FUNC_MACRO;
 		break;
@@ -162,7 +167,8 @@ TOKEN		[^[:space:](),]+
 }
 
 {TOKEN} {
-	yylval.str = xtalloc_strdup (yyextra, yytext);
+	yylval.token.type = TOKEN;
+	yylval.token.value = xtalloc_strdup (yyextra, yytext);
 	return TOKEN;
 }
 

glcpp-parse.y

@@ -108,16 +108,18 @@ glcpp_parser_lex (glcpp_parser_t *parser);
 	char *str;
 	argument_list_t *argument_list;
 	string_list_t *string_list;
+	token_t token;
 	token_list_t *token_list;
 }
 
 %parse-param {glcpp_parser_t *parser}
 %lex-param {glcpp_parser_t *parser}
 
-%token DEFINE FUNC_MACRO IDENTIFIER OBJ_MACRO NEWLINE SPACE TOKEN UNDEF
-%type <str> argument_word FUNC_MACRO IDENTIFIER OBJ_MACRO TOKEN
+%token DEFINE FUNC_MACRO IDENTIFIER IDENTIFIER_FINALIZED OBJ_MACRO NEWLINE SPACE TOKEN UNDEF
+%type <str> FUNC_MACRO IDENTIFIER IDENTIFIER_FINALIZED OBJ_MACRO
 %type <argument_list> argument_list
 %type <string_list> macro parameter_list
+%type <token> TOKEN argument_word
 %type <token_list> argument replacement_list pp_tokens
 
 /* Hard to remove shift/reduce conflicts documented as follows:
@@ -145,10 +147,14 @@ content:
 		printf ("%s", $1);
 		talloc_free ($1);
 	}
-|	TOKEN {
+|	IDENTIFIER_FINALIZED {
 		printf ("%s", $1);
 		talloc_free ($1);
 	}
+|	TOKEN {
+		printf ("%s", $1.value);
+		talloc_free ($1.value);
+	}
 |	FUNC_MACRO {
 		printf ("%s", $1);
 		talloc_free ($1);
@@ -189,11 +195,11 @@ argument_list:
 argument:
 	argument_word {
 		$$ = _token_list_create (parser);
-		_token_list_append ($$, IDENTIFIER, $1);
+		_token_list_append ($$, $1.type, $1.value);
 	}
 |	argument argument_word {
-		_token_list_append ($1, IDENTIFIER, $2);
-		talloc_free ($2);
+		_token_list_append ($1, $2.type, $2.value);
+		talloc_free ($2.value);
 		$$ = $1;
 	}
 |	argument '(' argument ')' {
@@ -205,10 +211,11 @@ argument:
 ;
 
 argument_word:
-	IDENTIFIER { $$ = $1; }
+	IDENTIFIER { $$.type = IDENTIFIER; $$.value = $1; }
+|	IDENTIFIER_FINALIZED { $$.type = IDENTIFIER_FINALIZED; $$.value = $1; }
 |	TOKEN { $$ = $1; }
-|	FUNC_MACRO { $$ = $1; }
-|	macro {	$$ = xtalloc_strdup (parser, ""); }
+|	FUNC_MACRO { $$.type = FUNC_MACRO; $$.value = $1; }
+|	macro {	$$.type = TOKEN; $$.value = xtalloc_strdup (parser, ""); }
 ;
 
 
@@ -265,10 +272,10 @@ replacement_list:
 pp_tokens:
 	TOKEN {
 		$$ = _token_list_create (parser);
-		_token_list_append ($$, TOKEN, $1);
+		_token_list_append ($$, $1.type, $1.value);
 	}
 |	pp_tokens TOKEN {
-	_token_list_append ($1, TOKEN, $2);
+	_token_list_append ($1, $2.type, $2.value);
 		$$ = $1;
 	}
 ;
@@ -567,7 +574,7 @@ glcpp_parser_classify_token (glcpp_parser_t *parser,
 	/* Don't consider this a macro if we are already actively
 	 * expanding this macro. */
 	if (glcpp_parser_is_expanding (parser, identifier))
-		return TOKEN_CLASS_IDENTIFIER;
+		return TOKEN_CLASS_IDENTIFIER_FINALIZED;
 
 	/* Definitely a macro. Just need to check if it's function-like. */
 	if (macro->is_function)
@@ -741,6 +748,10 @@ glcpp_parser_lex (glcpp_parser_t *parser)
 
 	yylval.str = xtalloc_strdup (parser, replacements->value);
 
+	/* Carefully refuse to expand any finalized identifier. */
+	if (replacements->type == IDENTIFIER_FINALIZED)
+		return IDENTIFIER_FINALIZED;
+
 	switch (glcpp_parser_classify_token (parser, yylval.str,
 					     &parameter_index))
 	{
@@ -753,6 +764,9 @@ glcpp_parser_lex (glcpp_parser_t *parser)
 	case TOKEN_CLASS_IDENTIFIER:
 		return IDENTIFIER;
 		break;
+	case TOKEN_CLASS_IDENTIFIER_FINALIZED:
+		return IDENTIFIER_FINALIZED;
+		break;
 	case TOKEN_CLASS_FUNC_MACRO:
 		return FUNC_MACRO;
 		break;

glcpp.h

@@ -42,6 +42,11 @@ typedef struct string_list {
 	string_node_t *tail;
 } string_list_t;
 
+typedef struct token {
+	int type;
+	char *value;
+} token_t;
+
 typedef struct token_node {
 	int type;
 	const char *value;
@@ -68,6 +73,7 @@ typedef struct glcpp_parser glcpp_parser_t;
 typedef enum {
 	TOKEN_CLASS_ARGUMENT,
 	TOKEN_CLASS_IDENTIFIER,
+	TOKEN_CLASS_IDENTIFIER_FINALIZED,
 	TOKEN_CLASS_FUNC_MACRO,
 	TOKEN_CLASS_OBJ_MACRO
 } token_class_t;

tests/037-finalize-unexpanded-macro.c

@@ -0,0 +1,3 @@
+#define expand(x) expand(x once)
+#define foo(x) x
+foo(expand(just))