Changes from Jonathan Riehl to allow his pgen extension (PEP 269) to

work.  This includes some more code that used to be part of pgen in
the main parser; I'm okay with that.  I'll see if the Windows build
needs work next.

Include/parsetok.h

@@ -38,6 +38,10 @@ PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
 					      const char *,
 					      grammar *, int,
                                               perrdetail *, int);
+
+/* Note that he following function is defined in pythonrun.c not parsetok.c. */
+PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
+
 #ifdef __cplusplus
 }
 #endif

Include/pgen.h

@@ -0,0 +1,18 @@
+#ifndef Py_PGEN_H
+#define Py_PGEN_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* Parser generator interface */
+
+extern grammar *meta_grammar(void);
+
+struct _node;
+extern grammar *pgen(struct _node *);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PGEN_H */

Makefile.pre.in

@@ -194,7 +194,10 @@ POBJS=		\
 		Parser/parser.o \
 		Parser/parsetok.o \
 		Parser/bitset.o \
-		Parser/metagrammar.o
+		Parser/metagrammar.o \
+		Parser/firstsets.o \
+		Parser/grammar.o \
+		Parser/pgen.o
 
 PARSER_OBJS=	$(POBJS) Parser/myreadline.o Parser/tokenizer.o
 
@@ -202,9 +205,6 @@ PGOBJS=		\
 		Objects/obmalloc.o \
 		Python/mysnprintf.o \
 		Parser/tokenizer_pgen.o \
-		Parser/firstsets.o \
-		Parser/grammar.o \
-		Parser/pgen.o \
 		Parser/printgrammar.o \
 		Parser/pgenmain.o
 

Parser/grammar.c

@@ -38,7 +38,7 @@ adddfa(grammar *g, int type, char *name)
 		Py_FatalError("no mem to resize dfa in adddfa");
 	d = &g->g_dfa[g->g_ndfas++];
 	d->d_type = type;
-	d->d_name = name;
+	d->d_name = strdup(name);
 	d->d_nstates = 0;
 	d->d_state = NULL;
 	d->d_initial = -1;
@@ -98,7 +98,10 @@ addlabel(labellist *ll, int type, char *str)
 		Py_FatalError("no mem to resize labellist in addlabel");
 	lb = &ll->ll_label[ll->ll_nlabels++];
 	lb->lb_type = type;
-	lb->lb_str = str; /* XXX strdup(str) ??? */
+	lb->lb_str = strdup(str);
+	if (Py_DebugFlag)
+		printf("Label @ %08x, %d: %s\n", (unsigned)ll, ll->ll_nlabels,
+		       PyGrammar_LabelRepr(lb));
 	return lb - ll->ll_label;
 }
 
@@ -152,6 +155,7 @@ translabel(grammar *g, label *lb)
 					    lb->lb_str,
 					    g->g_dfa[i].d_type);
 				lb->lb_type = g->g_dfa[i].d_type;
+				free(lb->lb_str);
 				lb->lb_str = NULL;
 				return;
 			}
@@ -162,6 +166,7 @@ translabel(grammar *g, label *lb)
 					printf("Label %s is terminal %d.\n",
 						lb->lb_str, i);
 				lb->lb_type = i;
+				free(lb->lb_str);
 				lb->lb_str = NULL;
 				return;
 			}
@@ -173,18 +178,29 @@ translabel(grammar *g, label *lb)
 	if (lb->lb_type == STRING) {
 		if (isalpha((int)(lb->lb_str[1])) || lb->lb_str[1] == '_') {
 			char *p;
+			char *src;
+			char *dest;
+			size_t name_len;
 			if (Py_DebugFlag)
 				printf("Label %s is a keyword\n", lb->lb_str);
 			lb->lb_type = NAME;
-			lb->lb_str++;
-			p = strchr(lb->lb_str, '\'');
+			src = lb->lb_str + 1;
+			p = strchr(src, '\'');
 			if (p)
-				*p = '\0';
+				name_len = p - src;
+			else
+				name_len = strlen(src);
+			dest = malloc(name_len + 1);
+			strncpy(dest, src, name_len);
+			dest[name_len] = '\0';
+			free(lb->lb_str);
+			lb->lb_str = dest;
 		}
 		else if (lb->lb_str[2] == lb->lb_str[0]) {
 			int type = (int) PyToken_OneChar(lb->lb_str[1]);
 			if (type != OP) {
 				lb->lb_type = type;
+				free(lb->lb_str);
 				lb->lb_str = NULL;
 			}
 			else
@@ -196,6 +212,7 @@ translabel(grammar *g, label *lb)
 						   lb->lb_str[2]);
 			if (type != OP) {
 				lb->lb_type = type;
+				free(lb->lb_str);
 				lb->lb_str = NULL;
 			}
 			else
@@ -208,6 +225,7 @@ translabel(grammar *g, label *lb)
 							    lb->lb_str[3]);
 			if (type != OP) {
 				lb->lb_type = type;
+				free(lb->lb_str);
 				lb->lb_str = NULL;
 			}
 			else

Parser/metagrammar.c

@@ -151,3 +151,9 @@ meta_grammar(void)
 {
 	return &_PyParser_Grammar;
 }
+
+grammar *
+Py_meta_grammar(void)
+{
+  return meta_grammar();
+}

Parser/pgen.c

@@ -669,6 +669,11 @@ pgen(node *n)
 	return g;
 }
 
+grammar *
+Py_pgen(node *n)
+{
+  return pgen(n);
+}
 
 /*