X-Git-Url: https://git.distorted.org.uk/~mdw/sgt/halibut/blobdiff_plain/b3c80b25d640d5641882482a2461980c90297274..e34ba5c3b8a7bcb8fceb437125da3a6a6f6d2dba:/input.c

diff --git a/input.c b/input.c
index 2e0dabf..d607e86 100644
--- a/input.c
+++ b/input.c
@@ -82,6 +82,16 @@ static void macrocleanup(tree234 *macros) {
     freetree234(macros);
 }
 
+static void input_configure(input *in, paragraph *cfg) {
+    assert(cfg->type == para_Config);
+
+    if (!ustricmp(cfg->keyword, L"input-charset")) {
+	char *csname = utoa_dup(uadv(cfg->keyword));
+	in->charset = charset_from_localenc(csname);
+	sfree(csname);
+    }
+}
+
 /*
  * Can return EOF
  */
@@ -103,36 +113,63 @@ static int get(input *in, filepos *pos) {
 	return c;
     }
     else if (in->currfp) {
-	int c = getc(in->currfp);
 
-	if (c == EOF) {
-	    fclose(in->currfp);
-	    in->currfp = NULL;
-	}
-	/* Track line numbers, for error reporting */
-	if (pos)
-	    *pos = in->pos;
-	if (in->reportcols) {
-	    switch (c) {
-	      case '\t':
-		in->pos.col = 1 + (in->pos.col + TAB_STOP-1) % TAB_STOP;
-		break;
-	      case '\n':
-		in->pos.col = 1;
-		in->pos.line++;
-		break;
-	      default:
-		in->pos.col++;
-		break;
+	while (in->wcpos >= in->nwc) {
+
+	    int c = getc(in->currfp);
+
+	    if (c == EOF) {
+		fclose(in->currfp);
+		in->currfp = NULL;
+		return EOF;
+	    }
+	    /* Track line numbers, for error reporting */
+	    if (pos)
+		*pos = in->pos;
+	    if (in->reportcols) {
+		switch (c) {
+		  case '\t':
+		    in->pos.col = 1 + (in->pos.col + TAB_STOP-1) % TAB_STOP;
+		    break;
+		  case '\n':
+		    in->pos.col = 1;
+		    in->pos.line++;
+		    break;
+		  default:
+		    in->pos.col++;
+		    break;
+		}
+	    } else {
+		in->pos.col = -1;
+		if (c == '\n')
+		    in->pos.line++;
+	    }
+
+	    /*
+	     * Do input character set translation, so that we return
+	     * Unicode.
+	     */
+	    {
+		char buf[1];
+		char const *p;
+		int inlen;
+
+		buf[0] = (char)c;
+		p = buf;
+		inlen = 1;
+
+		in->nwc = charset_to_unicode(&p, &inlen,
+					     in->wc, lenof(in->wc),
+					     in->charset, &in->csstate,
+					     NULL, 0);
+		assert(p == buf+1 && inlen == 0);
+
+		in->wcpos = 0;
 	    }
-	} else {
-	    in->pos.col = -1;
-	    if (c == '\n')
-		in->pos.line++;
 	}
-	/* FIXME: do input charmap translation. We should be returning
-	 * Unicode here. */
-	return c;
+
+	return in->wc[in->wcpos++];
+
     } else
 	return EOF;
 }
@@ -162,6 +199,7 @@ enum {
     c__invalid,			       /* invalid command */
     c__comment,			       /* comment command (\#) */
     c__escaped,			       /* escaped character */
+    c__nop,			       /* no-op */
     c__nbsp,			       /* nonbreaking space */
     c_A,			       /* appendix heading */
     c_B,			       /* bibliography entry */
@@ -180,15 +218,19 @@ enum {
     c_copyright,		       /* copyright statement */
     c_cw,			       /* weak code */
     c_date,			       /* document processing date */
+    c_dd,			       /* description list: description */
     c_define,			       /* macro definition */
+    c_dt,			       /* description list: described thing */
     c_e,			       /* emphasis */
     c_i,			       /* visible index mark */
     c_ii,			       /* uncapitalised visible index mark */
     c_k,			       /* uncapitalised cross-reference */
+    c_lcont,			       /* continuation para(s) for list item */
     c_n,			       /* numbered list */
     c_nocite,			       /* bibliography trickery */
-    c_preamble,			       /* document preamble text */
+    c_preamble,			       /* (obsolete) preamble text */
     c_q,			       /* quote marks */
+    c_quote,			       /* block-quoted paragraphs */
     c_rule,			       /* horizontal rule */
     c_title,			       /* document title */
     c_u,			       /* aux field is char code */
@@ -227,6 +269,7 @@ static void match_kw(token *tok) {
     static const struct { char const *name; int id; } keywords[] = {
 	{"#", c__comment},	       /* comment command (\#) */
 	{"-", c__escaped},	       /* nonbreaking hyphen */
+	{".", c__nop},		       /* no-op */
 	{"A", c_A},		       /* appendix heading */
 	{"B", c_B},		       /* bibliography entry */
 	{"BR", c_BR},		       /* bibliography rewrite */
@@ -245,15 +288,19 @@ static void match_kw(token *tok) {
 	{"copyright", c_copyright},    /* copyright statement */
 	{"cw", c_cw},		       /* weak code */
 	{"date", c_date},	       /* document processing date */
+	{"dd", c_dd},		       /* description list: description */
 	{"define", c_define},	       /* macro definition */
+	{"dt", c_dt},		       /* description list: described thing */
 	{"e", c_e},		       /* emphasis */
 	{"i", c_i},		       /* visible index mark */
 	{"ii", c_ii},		       /* uncapitalised visible index mark */
 	{"k", c_k},		       /* uncapitalised cross-reference */
+	{"lcont", c_lcont},	       /* continuation para(s) for list item */
 	{"n", c_n},		       /* numbered list */
 	{"nocite", c_nocite},	       /* bibliography trickery */
-	{"preamble", c_preamble},      /* document preamble text */
+	{"preamble", c_preamble},      /* (obsolete) preamble text */
 	{"q", c_q},		       /* quote marks */
+	{"quote", c_quote},	       /* block-quoted paragraphs */
 	{"rule", c_rule},	       /* horizontal rule */
 	{"title", c_title},	       /* document title */
 	{"versionid", c_versionid},    /* document RCS id */
@@ -352,7 +399,7 @@ token get_token(input *in) {
     } else if (c == '\\') {	       /* tok_cmd */
 	c = get(in, &cpos);
 	if (c == '-' || c == '\\' || c == '_' ||
-	    c == '#' || c == '{' || c == '}') {
+	    c == '#' || c == '{' || c == '}' || c == '.') {
 	    /* single-char command */
 	    rdadd(&rs, c);
 	} else if (c == 'u') {
@@ -506,6 +553,7 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
     int already;
     int iswhite, seenwhite;
     int type;
+    int prev_para_type;
     struct stack_item {
 	enum {
 	    stack_nop = 0,	       /* do nothing (for error recovery) */
@@ -517,8 +565,14 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 	} type;
 	word **whptr;		       /* to restore from \u alternatives */
 	word **idximplicit;	       /* to restore from \u alternatives */
+	filepos fpos;
     } *sitem;
     stack parsestk;
+    struct crossparaitem {
+ 	int type;		       /* currently c_lcont, c_quote or -1 */
+	int seen_lcont, seen_quote;
+    };
+    stack crossparastk;
     word *indexword, *uword, *iword;
     word *idxwordlist;
     rdstring indexstr;
@@ -530,6 +584,8 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
     macros = newtree234(macrocmp);
     already = FALSE;
 
+    crossparastk = stk_new();
+
     /*
      * Loop on each paragraph.
      */
@@ -542,22 +598,26 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 	/*
 	 * Get a token.
 	 */
-	if (!already) {
-	    dtor(t), t = get_token(in);
-	}
-	already = FALSE;
+	do {
+	    if (!already) {
+		dtor(t), t = get_token(in);
+	    }
+	    already = FALSE;
+	} while (t.type == tok_eop);
 	if (t.type == tok_eof)
-	    return;
+	    break;
 
 	/*
 	 * Parse code paragraphs separately.
 	 */
 	if (t.type == tok_cmd && t.cmd == c_c && !isbrace(in)) {
+	    int wtype = word_WeakCode;
+
 	    par.type = para_Code;
 	    par.fpos = t.pos;
 	    while (1) {
 		dtor(t), t = get_codepar_token(in);
-		wd.type = word_WeakCode;
+		wd.type = wtype;
 		wd.breaks = FALSE;     /* shouldn't need this... */
 		wd.text = ustrdup(t.text);
 		wd.alt = NULL;
@@ -570,22 +630,134 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 		     */
 		    dtor(t), t = get_token(in);
 		}
-		if (t.type == tok_eop || t.type == tok_eof)
+		if (t.type == tok_eop || t.type == tok_eof ||
+		    t.type == tok_rbrace) { /* might be } terminating \lcont */
+		    if (t.type == tok_rbrace)
+			already = TRUE;
 		    break;
-		else if (t.type != tok_cmd || t.cmd != c_c) {
+		} else if (t.type == tok_cmd && t.cmd == c_c) {
+		    wtype = word_WeakCode;
+		} else if (t.type == tok_cmd && t.cmd == c_e &&
+			   wtype == word_WeakCode) {
+		    wtype = word_Emph;
+		} else {
 		    error(err_brokencodepara, &t.pos);
+		    prev_para_type = par.type;
 		    addpara(par, ret);
 		    while (t.type != tok_eop)   /* error recovery: */
 			dtor(t), t = get_token(in);   /* eat rest of paragraph */
 		    goto codeparabroken;   /* ick, but such is life */
 		}
 	    }
+	    prev_para_type = par.type;
 	    addpara(par, ret);
 	    codeparabroken:
 	    continue;
 	}
 
 	/*
+	 * Spot the special commands that define a grouping of more
+	 * than one paragraph, and also the closing braces that
+	 * finish them.
+	 */
+	if (t.type == tok_cmd &&
+	    (t.cmd == c_lcont || t.cmd == c_quote)) {
+	    struct crossparaitem *sitem, *stop;
+	    int cmd = t.cmd;
+
+	    /*
+	     * Expect, and swallow, an open brace.
+	     */
+	    dtor(t), t = get_token(in);
+	    if (t.type != tok_lbrace) {
+		error(err_explbr, &t.pos);
+		continue;
+	    }
+
+	    /*
+	     * Also expect, and swallow, any whitespace after that
+	     * (a newline before a code paragraph wouldn't be
+	     * surprising).
+	     */
+	    do {
+		dtor(t), t = get_token(in);
+	    } while (t.type == tok_white);
+	    already = TRUE;
+
+	    if (cmd == c_lcont) {
+		/*
+		 * \lcont causes a continuation of a list item into
+		 * multiple paragraphs (which may in turn contain
+		 * nested lists, code paras etc). Hence, the previous
+		 * paragraph must be of a list type.
+		 */
+		sitem = mknew(struct crossparaitem);
+		stop = (struct crossparaitem *)stk_top(crossparastk);
+		if (stop)
+		    *sitem = *stop;
+		else
+		    sitem->seen_quote = sitem->seen_lcont = 0;
+
+		if (prev_para_type == para_Bullet ||
+		    prev_para_type == para_NumberedList ||
+		    prev_para_type == para_Description) {
+		    sitem->type = c_lcont;
+		    sitem->seen_lcont = 1;
+		    par.type = para_LcontPush;
+		    prev_para_type = par.type;
+		    addpara(par, ret);
+		} else {
+		    /*
+		     * Push a null item on the cross-para stack so that
+		     * when we see the corresponding closing brace we
+		     * don't give a cascade error.
+		     */
+		    sitem->type = -1;
+		    error(err_misplacedlcont, &t.pos);
+		}
+	    } else {
+		/*
+		 * \quote causes a group of paragraphs to be
+		 * block-quoted (typically they will be indented a
+		 * bit).
+		 */
+		sitem = mknew(struct crossparaitem);
+		stop = (struct crossparaitem *)stk_top(crossparastk);
+		if (stop)
+		    *sitem = *stop;
+		else
+		    sitem->seen_quote = sitem->seen_lcont = 0;
+		sitem->type = c_quote;
+		sitem->seen_quote = 1;
+		par.type = para_QuotePush;
+		prev_para_type = par.type;
+		addpara(par, ret);
+	    }
+	    stk_push(crossparastk, sitem);
+	    continue;
+	} else if (t.type == tok_rbrace) {
+	    struct crossparaitem *sitem = stk_pop(crossparastk);
+	    if (!sitem)
+		error(err_unexbrace, &t.pos);
+	    else {
+		switch (sitem->type) {
+		  case c_lcont:
+		    par.type = para_LcontPop;
+		    prev_para_type = par.type;
+		    addpara(par, ret);
+		    break;
+		  case c_quote:
+		    par.type = para_QuotePop;
+		    prev_para_type = par.type;
+		    addpara(par, ret);
+		    break;
+		}
+		sfree(sitem);
+	    }
+	    continue;
+	}
+
+	/*
 	 * This token begins a paragraph. See if it's one of the
 	 * special commands that define a paragraph type.
 	 *
@@ -638,6 +810,8 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 	      case c_U: needkw = 32; par.type = para_UnnumberedChapter; break;
 		/* For \b and \n the keyword is optional */
 	      case c_b: needkw = 4; par.type = para_Bullet; break;
+	      case c_dt: needkw = 4; par.type = para_DescribedThing; break;
+	      case c_dd: needkw = 4; par.type = para_Description; break;
 	      case c_n: needkw = 4; par.type = para_NumberedList; break;
 	      case c_cfg: needkw = 8; par.type = para_Config;
 		start_cmd = c_cfg; break;
@@ -645,12 +819,25 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 	      case c_define: is_macro = TRUE; needkw = 1; break;
 		/* For \nocite the keyword is _everything_ */
 	      case c_nocite: needkw = 8; par.type = para_NoCite; break;
-	      case c_preamble: needkw = 32; par.type = para_Preamble; break;
+	      case c_preamble: needkw = 32; par.type = para_Normal; break;
 	      case c_rule: needkw = 16; par.type = para_Rule; break;
 	      case c_title: needkw = 32; par.type = para_Title; break;
 	      case c_versionid: needkw = 32; par.type = para_VersionID; break;
 	    }
 
+	    if (par.type == para_Chapter ||
+		par.type == para_Heading ||
+		par.type == para_Subsect ||
+		par.type == para_Appendix ||
+		par.type == para_UnnumberedChapter) {
+		struct crossparaitem *sitem = stk_top(crossparastk);
+		if (sitem && (sitem->seen_lcont || sitem->seen_quote)) {
+		    error(err_sectmarkerinblock,
+			  &t.pos,
+			  (sitem->seen_lcont ? "lcont" : "quote"));
+		}
+	    }
+
 	    if (needkw > 0) {
 		rdstring rs = { 0, 0, NULL };
 		int nkeys = 0;
@@ -732,7 +919,12 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 		    }
 		    if (t.type == tok_cmd)
 			already = TRUE;/* inhibit get_token at top of loop */
+		    prev_para_type = par.type;
 		    addpara(par, ret);
+
+		    if (par.type == para_Config) {
+			input_configure(in, &par);
+		    }
 		    continue;	       /* next paragraph */
 		}
 	    }
@@ -770,6 +962,11 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 		break;
 	    }
 
+	    if (t.type == tok_cmd && t.cmd == c__nop) {
+		dtor(t), t = get_token(in);
+		continue;	       /* do nothing! */
+	    }
+
 	    if (t.type == tok_cmd && t.cmd == c__escaped) {
 		t.type = tok_word;     /* nice and simple */
 		t.aux = 0;	       /* even if `\-' - nonbreaking! */
@@ -833,13 +1030,21 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 		/* Error recovery: push nop */
 		sitem = mknew(struct stack_item);
 		sitem->type = stack_nop;
+		sitem->fpos = t.pos;
 		stk_push(parsestk, sitem);
 		break;
 	      case tok_rbrace:
 		sitem = stk_pop(parsestk);
-		if (!sitem)
-		    error(err_unexbrace, &t.pos);
-		else {
+		if (!sitem) {
+		    /*
+		     * This closing brace could have been an
+		     * indication that the cross-paragraph stack
+		     * wants popping. Accordingly, we treat it here
+		     * as an indication that the paragraph is over.
+		     */
+		    already = TRUE;
+		    goto finished_para;
+		} else {
 		    if (sitem->type & stack_ualt) {
 			whptr = sitem->whptr;
 			idximplicit = sitem->idximplicit;
@@ -850,11 +1055,20 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 		    }
 		    if (sitem->type & stack_idx) {
 			indexword->text = ustrdup(indexstr.text);
-			if (index_downcase)
+			if (index_downcase) {
+			    word *w;
+
 			    ustrlow(indexword->text);
+			    ustrlow(indexstr.text);
+
+			    for (w = idxwordlist; w; w = w->next)
+				if (w->text)
+				    ustrlow(w->text);
+			}
 			indexing = FALSE;
 			rdadd(&indexstr, L'\0');
-			index_merge(idx, FALSE, indexstr.text, idxwordlist);
+			index_merge(idx, FALSE, indexstr.text,
+				    idxwordlist, &sitem->fpos);
 			sfree(indexstr.text);
 		    }
 		    if (sitem->type & stack_hyper) {
@@ -940,6 +1154,7 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 			    addword(wd, &idximplicit);
 			}
 			sitem = mknew(struct stack_item);
+			sitem->fpos = t.pos;
 			sitem->type = stack_quote;
 			stk_push(parsestk, sitem);
 		    }
@@ -1015,11 +1230,43 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 			 * delimiting the text marked by the link.
 			 */
 			dtor(t), t = get_token(in);
+			sitem = mknew(struct stack_item);
+			sitem->fpos = wd.fpos;
+			sitem->type = stack_hyper;
+			/*
+			 * Special cases: \W{}\i, \W{}\ii
+			 */
+			if (t.type == tok_cmd &&
+			    (t.cmd == c_i || t.cmd == c_ii)) {
+			    if (indexing) {
+				error(err_nestedindex, &t.pos);
+			    } else {
+				/* Add an index-reference word with no
+				 * text as yet */
+				wd.type = word_IndexRef;
+				wd.text = NULL;
+				wd.alt = NULL;
+				wd.aux = 0;
+				wd.breaks = FALSE;
+				indexword = addword(wd, &whptr);
+				/* Set up a rdstring to read the
+				 * index text */
+				indexstr = nullrs;
+				/* Flags so that we do the Right
+				 * Things with text */
+				index_visible = (type != c_I);
+				index_downcase = (type == c_ii);
+				indexing = TRUE;
+				idxwordlist = NULL;
+				idximplicit = &idxwordlist;
+
+				sitem->type |= stack_idx;
+			    }
+			    dtor(t), t = get_token(in);
+			}
 			/*
 			 * Special cases: \W{}\c, \W{}\e, \W{}\cw
 			 */
-			sitem = mknew(struct stack_item);
-			sitem->type = stack_hyper;
 			if (t.type == tok_cmd &&
 			    (t.cmd == c_e || t.cmd == c_c || t.cmd == c_cw)) {
 			    if (style != word_Normal)
@@ -1050,6 +1297,7 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 			/* Error recovery: eat lbrace, push nop. */
 			dtor(t), t = get_token(in);
 			sitem = mknew(struct stack_item);
+			sitem->fpos = t.pos;
 			sitem->type = stack_nop;
 			stk_push(parsestk, sitem);
 		    }
@@ -1062,6 +1310,7 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 				 word_Emph);
 			spcstyle = tospacestyle(style);
 			sitem = mknew(struct stack_item);
+			sitem->fpos = t.pos;
 			sitem->type = stack_style;
 			stk_push(parsestk, sitem);
 		    }
@@ -1075,10 +1324,12 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 			/* Error recovery: eat lbrace, push nop. */
 			dtor(t), t = get_token(in);
 			sitem = mknew(struct stack_item);
+			sitem->fpos = t.pos;
 			sitem->type = stack_nop;
 			stk_push(parsestk, sitem);
 		    }
 		    sitem = mknew(struct stack_item);
+		    sitem->fpos = t.pos;
 		    sitem->type = stack_idx;
 		    dtor(t), t = get_token(in);
 		    /*
@@ -1148,6 +1399,7 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 			 * paragraph.
 			 */
 			sitem = mknew(struct stack_item);
+			sitem->fpos = t.pos;
 			sitem->type = stack_ualt;
 			sitem->whptr = whptr;
 			sitem->idximplicit = idximplicit;
@@ -1170,19 +1422,36 @@ static void read_file(paragraph ***ret, input *in, indexdata *idx) {
 		dtor(t), t = get_token(in);
 	    seenwhite = iswhite;
 	}
+	finished_para:
 	/* Check the stack is empty */
-	if (NULL != (sitem = stk_pop(parsestk))) {
-	    do {
+	if (stk_top(parsestk)) {
+	    while ((sitem = stk_pop(parsestk)))
 		sfree(sitem);
-		sitem = stk_pop(parsestk);
-	    } while (sitem);
 	    error(err_missingrbrace, &t.pos);
 	}
 	stk_free(parsestk);
+	prev_para_type = par.type;
 	addpara(par, ret);
+	if (t.type == tok_eof)
+	    already = TRUE;
     }
+
+    if (stk_top(crossparastk)) {
+	void *p;
+
+	error(err_missingrbrace2, &t.pos);
+	while ((p = stk_pop(crossparastk)))
+	    sfree(p);
+    }
+
+    /*
+     * We break to here rather than returning, because otherwise
+     * this cleanup doesn't happen.
+     */
     dtor(t);
     macrocleanup(macros);
+
+    stk_free(crossparastk);
 }
 
 paragraph *read_input(input *in, indexdata *idx) {
@@ -1193,6 +1462,8 @@ paragraph *read_input(input *in, indexdata *idx) {
 	in->currfp = fopen(in->filenames[in->currindex], "r");
 	if (in->currfp) {
 	    setpos(in, in->filenames[in->currindex]);
+	    in->charset = in->defcharset;
+	    in->csstate = charset_init_state;
 	    read_file(&hptr, in, idx);
 	}
 	in->currindex++;