Re-engineering of terminal emulator, phase 1.
[u/mdw/putty] / terminal.c
index e3e8cc0..d091162 100644 (file)
@@ -56,6 +56,8 @@
 
 #define has_compat(x) ( ((CL_##x)&term->compatibility_level) != 0 )
 
+const char sco2ansicolour[] = { 0, 4, 2, 6, 1, 5, 3, 7 };
+
 #define sel_nl_sz  (sizeof(sel_nl)/sizeof(wchar_t))
 const wchar_t sel_nl[] = SEL_NL;
 
@@ -66,14 +68,25 @@ const wchar_t sel_nl[] = SEL_NL;
  * then we must look one space further to the left.
  */
 #define UCSGET(a, x) \
-    ( (x)>0 && ((a)[(x)] & (CHAR_MASK | CSET_MASK)) == UCSWIDE ? \
-       (a)[(x)-1] : (a)[(x)] )
+    ( (x)>0 && (a)[(x)].chr == UCSWIDE ? (a)[(x)-1].chr : (a)[(x)].chr )
+
+/*
+ * Detect the various aliases of U+0020 SPACE.
+ */
+#define IS_SPACE_CHR(chr) \
+       ((chr) == 0x20 || (DIRECT_CHAR(chr) && ((chr) & 0xFF) == 0x20))
+
+/*
+ * Spot magic CSETs.
+ */
+#define CSET_OF(chr) (DIRECT_CHAR(chr)||DIRECT_FONT(chr) ? (chr)&CSET_MASK : 0)
 
 /*
  * Internal prototypes.
  */
-static unsigned long *resizeline(unsigned long *, int);
-static unsigned long *lineptr(Terminal *, int, int);
+static void resizeline(Terminal *, termline *, int);
+static termline *lineptr(Terminal *, int, int, int);
+static void unlineptr(termline *);
 static void do_paint(Terminal *, Context, int);
 static void erase_lots(Terminal *, int, int, int);
 static void swap_screen(Terminal *, int, int, int);
@@ -84,29 +97,524 @@ static void term_print_finish(Terminal *);
 static void scroll_display(Terminal *, int, int, int);
 #endif /* OPTIMISE_SCROLL */
 
+static termline *newline(Terminal *term, int cols, int bce)
+{
+    termline *line;
+    int j;
+
+    line = snew(termline);
+    line->chars = snewn(cols, termchar);
+    for (j = 0; j < cols; j++)
+       line->chars[j] = (bce ? term->erase_char : term->basic_erase_char);
+    line->cols = cols;
+    line->lattr = LATTR_NORM;
+    line->temporary = FALSE;
+
+    return line;
+}
+
+static void freeline(termline *line)
+{
+    sfree(line->chars);
+    sfree(line);
+}
+
+static void unlineptr(termline *line)
+{
+    if (line->temporary)
+       freeline(line);
+}
+
+/*
+ * Compress and decompress a termline into an RLE-based format for
+ * storing in scrollback. (Since scrollback almost never needs to
+ * be modified and exists in huge quantities, this is a sensible
+ * tradeoff, particularly since it allows us to continue adding
+ * features to the main termchar structure without proportionally
+ * bloating the terminal emulator's memory footprint unless those
+ * features are in constant use.)
+ */
+struct buf {
+    unsigned char *data;
+    int len, size;
+};
+static void add(struct buf *b, unsigned char c)
+{
+    if (b->len >= b->size) {
+       b->size = (b->len * 3 / 2) + 512;
+       b->data = sresize(b->data, b->size, unsigned char);
+    }
+    b->data[b->len++] = c;
+}
+static int get(struct buf *b)
+{
+    return b->data[b->len++];
+}
+static void makerle(struct buf *b, termline *ldata,
+                   void (*makeliteral)(struct buf *b, termchar *c,
+                                       unsigned long *state))
+{
+    int hdrpos, hdrsize, n, prevlen, prevpos, thislen, thispos, prev2;
+    termchar *c = ldata->chars;
+    unsigned long state = 0, oldstate;
+
+    n = ldata->cols;
+
+    hdrpos = b->len;
+    hdrsize = 0;
+    add(b, 0);
+    prevlen = prevpos = 0;
+    prev2 = FALSE;
+
+    while (n-- > 0) {
+       thispos = b->len;
+       makeliteral(b, c++, &state);
+       thislen = b->len - thispos;
+       if (thislen == prevlen &&
+           !memcmp(b->data + prevpos, b->data + thispos, thislen)) {
+           /*
+            * This literal precisely matches the previous one.
+            * Turn it into a run if it's worthwhile.
+            * 
+            * With one-byte literals, it costs us two bytes to
+            * encode a run, plus another byte to write the header
+            * to resume normal output; so a three-element run is
+            * neutral, and anything beyond that is unconditionally
+            * worthwhile. With two-byte literals or more, even a
+            * 2-run is a win.
+            */
+           if (thislen > 1 || prev2) {
+               int runpos, runlen;
+
+               /*
+                * It's worth encoding a run. Start at prevpos,
+                * unless hdrsize==0 in which case we can back up
+                * another one and start by overwriting hdrpos.
+                */
+
+               hdrsize--;             /* remove the literal at prevpos */
+               if (prev2) {
+                   assert(hdrsize > 0);
+                   hdrsize--;
+                   prevpos -= prevlen;/* and possibly another one */
+               }
+
+               if (hdrsize == 0) {
+                   assert(prevpos == hdrpos + 1);
+                   runpos = hdrpos;
+                   b->len = prevpos+prevlen;
+               } else {
+                   memmove(b->data + prevpos+1, b->data + prevpos, prevlen);
+                   runpos = prevpos;
+                   b->len = prevpos+prevlen+1;
+                   /*
+                    * Terminate the previous run of ordinary
+                    * literals.
+                    */
+                   assert(hdrsize >= 1 && hdrsize <= 128);
+                   b->data[hdrpos] = hdrsize - 1;
+               }
+
+               runlen = prev2 ? 3 : 2;
+
+               while (n > 0 && runlen < 129) {
+                   int tmppos, tmplen;
+                   tmppos = b->len;
+                   oldstate = state;
+                   makeliteral(b, c, &state);
+                   tmplen = b->len - tmppos;
+                   b->len = tmppos;
+                   if (tmplen != thislen ||
+                       memcmp(b->data + runpos+1, b->data + tmppos, tmplen)) {
+                       state = oldstate;
+                       break;         /* run over */
+                   }
+                   n--, c++, runlen++;
+               }
+
+               assert(runlen >= 2 && runlen <= 129);
+               b->data[runpos] = runlen + 0x80 - 2;
+
+               hdrpos = b->len;
+               hdrsize = 0;
+               add(b, 0);
+
+               continue;
+           } else {
+               /*
+                * Just flag that the previous two literals were
+                * identical, in case we find a third identical one
+                * we want to turn into a run.
+                */
+               prev2 = TRUE;
+               prevlen = thislen;
+               prevpos = thispos;
+           }
+       } else {
+           prev2 = FALSE;
+           prevlen = thislen;
+           prevpos = thispos;
+       }
+
+       /*
+        * This character isn't (yet) part of a run. Add it to
+        * hdrsize.
+        */
+       hdrsize++;
+       if (hdrsize == 128) {
+           b->data[hdrpos] = hdrsize - 1;
+           hdrpos = b->len;
+           hdrsize = 0;
+           add(b, 0);
+           prevlen = prevpos = 0;
+           prev2 = FALSE;
+       }
+    }
+
+    /*
+     * Clean up.
+     */
+    if (hdrsize > 0) {
+       assert(hdrsize <= 128);
+       b->data[hdrpos] = hdrsize - 1;
+    } else {
+       b->len = hdrpos;
+    }
+}
+static void makeliteral_chr(struct buf *b, termchar *c, unsigned long *state)
+{
+    /*
+     * My encoding for characters is UTF-8-like, in that it stores
+     * 7-bit ASCII in one byte and uses high-bit-set bytes as
+     * introducers to indicate a longer sequence. However, it's
+     * unlike UTF-8 in that it doesn't need to be able to
+     * resynchronise, and therefore I don't want to waste two bits
+     * per byte on having recognisable continuation characters.
+     * Also I don't want to rule out the possibility that I may one
+     * day use values 0x80000000-0xFFFFFFFF for interesting
+     * purposes, so unlike UTF-8 I need a full 32-bit range.
+     * Accordingly, here is my encoding:
+     * 
+     * 00000000-0000007F: 0xxxxxxx (but see below)
+     * 00000080-00003FFF: 10xxxxxx xxxxxxxx
+     * 00004000-001FFFFF: 110xxxxx xxxxxxxx xxxxxxxx
+     * 00200000-0FFFFFFF: 1110xxxx xxxxxxxx xxxxxxxx xxxxxxxx
+     * 10000000-FFFFFFFF: 11110ZZZ xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
+     * 
+     * (`Z' is like `x' but is always going to be zero since the
+     * values I'm encoding don't go above 2^32. In principle the
+     * five-byte form of the encoding could extend to 2^35, and
+     * there could be six-, seven-, eight- and nine-byte forms as
+     * well to allow up to 64-bit values to be encoded. But that's
+     * completely unnecessary for these purposes!)
+     * 
+     * The encoding as written above would be very simple, except
+     * that 7-bit ASCII can occur in several different ways in the
+     * terminal data; sometimes it crops up in the D800 page
+     * (CSET_ASCII) but at other times it's in the 0000 page (real
+     * Unicode). Therefore, this encoding is actually _stateful_:
+     * the one-byte encoding of 00-7F actually indicates `reuse the
+     * upper three bytes of the last character', and to encode an
+     * absolute value of 00-7F you need to use the two-byte form
+     * instead.
+     */
+    if ((c->chr & ~0x7F) == *state) {
+       add(b, (unsigned char)(c->chr & 0x7F));
+    } else if (c->chr < 0x4000) {
+       add(b, (unsigned char)(((c->chr >> 8) & 0x3F) | 0x80));
+       add(b, (unsigned char)(c->chr & 0xFF));
+    } else if (c->chr < 0x200000) {
+       add(b, (unsigned char)(((c->chr >> 16) & 0x1F) | 0xC0));
+       add(b, (unsigned char)((c->chr >> 8) & 0xFF));
+       add(b, (unsigned char)(c->chr & 0xFF));
+    } else if (c->chr < 0x10000000) {
+       add(b, (unsigned char)(((c->chr >> 24) & 0x0F) | 0xE0));
+       add(b, (unsigned char)((c->chr >> 16) & 0xFF));
+       add(b, (unsigned char)((c->chr >> 8) & 0xFF));
+       add(b, (unsigned char)(c->chr & 0xFF));
+    } else {
+       add(b, 0xF0);
+       add(b, (unsigned char)((c->chr >> 24) & 0xFF));
+       add(b, (unsigned char)((c->chr >> 16) & 0xFF));
+       add(b, (unsigned char)((c->chr >> 8) & 0xFF));
+       add(b, (unsigned char)(c->chr & 0xFF));
+    }
+    *state = c->chr & ~0xFF;
+}
+static void makeliteral_attr(struct buf *b, termchar *c, unsigned long *state)
+{
+    /*
+     * My encoding for attributes is 16-bit-granular and assumes
+     * that the top bit of the word is never required. I either
+     * store a two-byte value with the top bit clear (indicating
+     * just that value), or a four-byte value with the top bit set
+     * (indicating the same value with its top bit clear).
+     */
+    if (c->attr < 0x8000) {
+       add(b, (unsigned char)((c->attr >> 8) & 0xFF));
+       add(b, (unsigned char)(c->attr & 0xFF));
+    } else {
+       add(b, (unsigned char)(((c->attr >> 24) & 0x7F) | 0x80));
+       add(b, (unsigned char)((c->attr >> 16) & 0xFF));
+       add(b, (unsigned char)((c->attr >> 8) & 0xFF));
+       add(b, (unsigned char)(c->attr & 0xFF));
+    }
+}
+
+static termline *decompressline(unsigned char *data, int *bytes_used);
+
+static unsigned char *compressline(termline *ldata)
+{
+    struct buf buffer = { NULL, 0, 0 }, *b = &buffer;
+
+    /*
+     * First, store the column count, 7 bits at a time, least
+     * significant `digit' first, with the high bit set on all but
+     * the last.
+     */
+    {
+       int n = ldata->cols;
+       while (n >= 128) {
+           add(b, (unsigned char)((n & 0x7F) | 0x80));
+           n >>= 7;
+       }
+       add(b, (unsigned char)(n));
+    }
+
+    /*
+     * Next store the lattrs; same principle.
+     */
+    {
+       int n = ldata->lattr;
+       while (n >= 128) {
+           add(b, (unsigned char)((n & 0x7F) | 0x80));
+           n >>= 7;
+       }
+       add(b, (unsigned char)(n));
+    }
+
+    /*
+     * Now we store a sequence of separate run-length encoded
+     * fragments, each containing exactly as many symbols as there
+     * are columns in the ldata.
+     * 
+     * All of these have a common basic format:
+     * 
+     *  - a byte 00-7F indicates that X+1 literals follow it
+     *         - a byte 80-FF indicates that a single literal follows it
+     *           and expects to be repeated (X-0x80)+2 times.
+     * 
+     * The format of the `literals' varies between the fragments.
+     */
+    makerle(b, ldata, makeliteral_chr);
+    makerle(b, ldata, makeliteral_attr);
+
+    /*
+     * Diagnostics: ensure that the compressed data really does
+     * decompress to the right thing.
+     */
+#ifndef CHECK_SB_COMPRESSION
+    {
+       int dused;
+       termline *dcl;
+
+#ifdef DIAGNOSTIC_SB_COMPRESSION
+       int i;
+       for (i = 0; i < b->len; i++) {
+           printf(" %02x ", b->data[i]);
+       }
+       printf("\n");
+#endif
+
+       dcl = decompressline(b->data, &dused);
+       assert(b->len == dused);
+       assert(ldata->cols == dcl->cols);
+       assert(ldata->lattr == dcl->lattr);
+       assert(!memcmp(ldata->chars, dcl->chars, ldata->cols * TSIZE));
+
+#ifdef DIAGNOSTIC_SB_COMPRESSION
+       printf("%d cols (%d bytes) -> %d bytes (factor of %g)\n",
+              ldata->cols, 4 * ldata->cols, dused,
+              (double)dused / (4 * ldata->cols));
+#endif
+
+       freeline(dcl);
+    }
+#endif
+
+    /*
+     * Trim the allocated memory so we don't waste any, and return.
+     */
+    return sresize(b->data, b->len, unsigned char);
+}
+
+static void readrle(struct buf *b, termline *ldata,
+                   void (*readliteral)(struct buf *b, termchar *c,
+                                       unsigned long *state))
+{
+    int n = 0;
+    unsigned long state = 0;
+
+    while (n < ldata->cols) {
+       int hdr = get(b);
+
+       if (hdr >= 0x80) {
+           /* A run. */
+
+           int pos = b->len, count = hdr + 2 - 0x80;
+           while (count--) {
+               assert(n < ldata->cols);
+               b->len = pos;
+               readliteral(b, ldata->chars + n, &state);
+               n++;
+           }
+       } else {
+           /* Just a sequence of consecutive literals. */
+
+           int count = hdr + 1;
+           while (count--) {
+               assert(n < ldata->cols);
+               readliteral(b, ldata->chars + n, &state);
+               n++;
+           }
+       }
+    }
+
+    assert(n == ldata->cols);
+}
+static void readliteral_chr(struct buf *b, termchar *c, unsigned long *state)
+{
+    int byte;
+
+    /*
+     * 00000000-0000007F: 0xxxxxxx
+     * 00000080-00003FFF: 10xxxxxx xxxxxxxx
+     * 00004000-001FFFFF: 110xxxxx xxxxxxxx xxxxxxxx
+     * 00200000-0FFFFFFF: 1110xxxx xxxxxxxx xxxxxxxx xxxxxxxx
+     * 10000000-FFFFFFFF: 11110ZZZ xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
+     */
+
+    byte = get(b);
+    if (byte < 0x80) {
+       c->chr = byte | *state;
+    } else if (byte < 0xC0) {
+       c->chr = (byte &~ 0xC0) << 8;
+       c->chr |= get(b);
+    } else if (byte < 0xE0) {
+       c->chr = (byte &~ 0xE0) << 16;
+       c->chr |= get(b) << 8;
+       c->chr |= get(b);
+    } else if (byte < 0xF0) {
+       c->chr = (byte &~ 0xF0) << 24;
+       c->chr |= get(b) << 16;
+       c->chr |= get(b) << 8;
+       c->chr |= get(b);
+    } else {
+       assert(byte == 0xF0);
+       c->chr = get(b) << 24;
+       c->chr |= get(b) << 16;
+       c->chr |= get(b) << 8;
+       c->chr |= get(b);
+    }
+    *state = c->chr & ~0xFF;
+}
+static void readliteral_attr(struct buf *b, termchar *c, unsigned long *state)
+{
+    int val;
+
+    val = get(b) << 8;
+    val |= get(b);
+
+    if (val >= 0x8000) {
+       val <<= 16;
+       val |= get(b) << 8;
+       val |= get(b);
+    }
+
+    c->attr = val;
+}
+
+static termline *decompressline(unsigned char *data, int *bytes_used)
+{
+    int ncols, byte, shift;
+    struct buf buffer, *b = &buffer;
+    termline *ldata;
+
+    b->data = data;
+    b->len = 0;
+
+    /*
+     * First read in the column count.
+     */
+    ncols = shift = 0;
+    do {
+       byte = get(b);
+       ncols |= (byte & 0x7F) << shift;
+       shift += 7;
+    } while (byte & 0x80);
+
+    /*
+     * Now create the output termline.
+     */
+    ldata = snew(termline);
+    ldata->chars = snewn(ncols, termchar);
+    ldata->cols = ncols;
+    ldata->temporary = TRUE;
+
+    /*
+     * Now read in the lattr.
+     */
+    ldata->lattr = shift = 0;
+    do {
+       byte = get(b);
+       ldata->lattr |= (byte & 0x7F) << shift;
+       shift += 7;
+    } while (byte & 0x80);
+
+    /*
+     * Now we read in each of the RLE streams in turn.
+     */
+    readrle(b, ldata, readliteral_chr);
+    readrle(b, ldata, readliteral_attr);
+
+    /* Return the number of bytes read, for diagnostic purposes. */
+    if (bytes_used)
+       *bytes_used = b->len;
+
+    return ldata;
+}
+
 /*
  * Resize a line to make it `cols' columns wide.
  */
-static unsigned long *resizeline(unsigned long *line, int cols)
+static void resizeline(Terminal *term, termline *line, int cols)
 {
     int i, oldlen;
-    unsigned long lineattrs;
 
-    if (line[0] != (unsigned long)cols) {
+    if (line->cols != cols) {
        /*
         * This line is the wrong length, which probably means it
         * hasn't been accessed since a resize. Resize it now.
         */
-       oldlen = line[0];
-       lineattrs = line[oldlen + 1];
-       line = srealloc(line, TSIZE * (2 + cols));
-       line[0] = cols;
+       oldlen = line->cols;
+       line->chars = sresize(line->chars, cols, TTYPE);
+       line->cols = cols;
        for (i = oldlen; i < cols; i++)
-           line[i + 1] = ERASE_CHAR;
-       line[cols + 1] = lineattrs & LATTR_MODE;
+           line->chars[i] = term->basic_erase_char;
     }
+}
 
-    return line;
+/*
+ * Get the number of lines in the scrollback.
+ */
+static int sblines(Terminal *term)
+{
+    int sblines = count234(term->scrollback);
+    if (term->cfg.erase_to_scrollback &&
+       term->alt_which && term->alt_screen) {
+           sblines += term->alt_sblines;
+    }
+    return sblines;
 }
 
 /*
@@ -114,9 +622,9 @@ static unsigned long *resizeline(unsigned long *line, int cols)
  * whether the y coordinate is non-negative or negative
  * (respectively).
  */
-static unsigned long *lineptr(Terminal *term, int y, int lineno)
+static termline *lineptr(Terminal *term, int y, int lineno, int screen)
 {
-    unsigned long *line, *newline;
+    termline *line;
     tree234 *whichtree;
     int treeindex;
 
@@ -124,25 +632,41 @@ static unsigned long *lineptr(Terminal *term, int y, int lineno)
        whichtree = term->screen;
        treeindex = y;
     } else {
-       whichtree = term->scrollback;
-       treeindex = y + count234(term->scrollback);
+       int altlines = 0;
+
+       assert(!screen);
+
+       if (term->cfg.erase_to_scrollback &&
+           term->alt_which && term->alt_screen) {
+           altlines = term->alt_sblines;
+       }
+       if (y < -altlines) {
+           whichtree = term->scrollback;
+           treeindex = y + altlines + count234(term->scrollback);
+       } else {
+           whichtree = term->alt_screen;
+           treeindex = y + term->alt_sblines;
+           /* treeindex = y + count234(term->alt_screen); */
+       }
+    }
+    if (whichtree == term->scrollback) {
+       unsigned char *cline = index234(whichtree, treeindex);
+       line = decompressline(cline, NULL);
+    } else {
+       line = index234(whichtree, treeindex);
     }
-    line = index234(whichtree, treeindex);
 
     /* We assume that we don't screw up and retrieve something out of range. */
     assert(line != NULL);
 
-    newline = resizeline(line, term->cols);
-    if (newline != line) {
-       delpos234(whichtree, treeindex);
-       addpos234(whichtree, newline, treeindex);
-        line = newline;
-    }
+    resizeline(term, line, term->cols);
+    /* FIXME: should we sort the compressed scrollback out here? */
 
-    return line + 1;
+    return line;
 }
 
-#define lineptr(x) lineptr(term,x,__LINE__)
+#define lineptr(x) (lineptr)(term,x,__LINE__,FALSE)
+#define scrlineptr(x) (lineptr)(term,x,__LINE__,TRUE)
 
 /*
  * Set up power-on settings for the terminal.
@@ -170,18 +694,18 @@ static void power_on(Terminal *term)
     term->alt_utf = term->utf = term->save_utf = 0;
     term->utf_state = 0;
     term->alt_sco_acs = term->sco_acs = term->save_sco_acs = 0;
-    term->cset_attr[0] = term->cset_attr[1] = term->save_csattr = ATTR_ASCII;
+    term->cset_attr[0] = term->cset_attr[1] = term->save_csattr = CSET_ASCII;
     term->rvideo = 0;
     term->in_vbell = FALSE;
     term->cursor_on = 1;
     term->big_cursor = 0;
-    term->save_attr = term->curr_attr = ATTR_DEFAULT;
+    term->default_attr = term->save_attr = term->curr_attr = ATTR_DEFAULT;
     term->term_editing = term->term_echoing = FALSE;
     term->app_cursor_keys = term->cfg.app_cursor;
     term->app_keypad_keys = term->cfg.app_keypad;
     term->use_bce = term->cfg.bce;
     term->blink_is_real = term->cfg.blinktext;
-    term->erase_char = ERASE_CHAR;
+    term->erase_char = term->basic_erase_char;
     term->alt_which = 0;
     term_print_finish(term);
     {
@@ -211,6 +735,15 @@ void term_update(Terminal *term)
            term->seen_disp_event = 0;
            need_sbar_update = TRUE;
        }
+
+       /* Allocate temporary buffers for Arabic shaping and bidi. */
+       if (!term->cfg.arabicshaping || !term->cfg.bidi)
+       {
+           term->wcFrom = sresize(term->wcFrom, term->cols, bidi_char);
+           term->ltemp = sresize(term->ltemp, term->cols+1, termchar);
+           term->wcTo = sresize(term->wcTo, term->cols, bidi_char);
+       }
+
        if (need_sbar_update)
            update_sbar(term);
        do_paint(term, ctx, TRUE);
@@ -264,6 +797,14 @@ void term_pwron(Terminal *term)
     term_update(term);
 }
 
+static void set_erase_char(Terminal *term)
+{
+    term->erase_char = term->basic_erase_char;
+    if (term->use_bce)
+       term->erase_char.attr = (term->curr_attr &
+                                (ATTR_FGMASK | ATTR_BGMASK));
+}
+
 /*
  * When the user reconfigures us, we need to check the forbidden-
  * alternate-screen config option, disable raw mouse mode if the
@@ -297,8 +838,10 @@ void term_reconfig(Terminal *term, Config *cfg)
        term->alt_wrap = term->wrap = term->cfg.wrap_mode;
     if (reset_decom)
        term->alt_om = term->dec_om = term->cfg.dec_om;
-    if (reset_bce)
+    if (reset_bce) {
        term->use_bce = term->cfg.bce;
+       set_erase_char(term);
+    }
     if (reset_blink)
        term->blink_is_real = term->cfg.blinktext;
     if (reset_charclass)
@@ -312,7 +855,7 @@ void term_reconfig(Terminal *term, Config *cfg)
        set_raw_mouse_mode(term->frontend, 0);
     }
     if (term->cfg.no_remote_charset) {
-       term->cset_attr[0] = term->cset_attr[1] = ATTR_ASCII;
+       term->cset_attr[0] = term->cset_attr[1] = CSET_ASCII;
        term->sco_acs = term->alt_sco_acs = 0;
        term->utf = 0;
     }
@@ -326,11 +869,13 @@ void term_reconfig(Terminal *term, Config *cfg)
  */
 void term_clrsb(Terminal *term)
 {
-    unsigned long *line;
+    termline *line;
     term->disptop = 0;
     while ((line = delpos234(term->scrollback, 0)) != NULL) {
-       sfree(line);
+       sfree(line);            /* this is compressed data, not a termline */
     }
+    term->tempsblines = 0;
+    term->alt_sblines = 0;
     update_sbar(term);
 }
 
@@ -346,7 +891,7 @@ Terminal *term_init(Config *mycfg, struct unicode_data *ucsdata,
      * Allocate a new Terminal structure and initialise the fields
      * that need it.
      */
-    term = smalloc(sizeof(Terminal));
+    term = snew(Terminal);
     term->frontend = frontend;
     term->ucsdata = ucsdata;
     term->cfg = *mycfg;                       /* STRUCTURE COPY */
@@ -374,37 +919,58 @@ Terminal *term_init(Config *mycfg, struct unicode_data *ucsdata,
     term->curstype = 0;
 
     term->screen = term->alt_screen = term->scrollback = NULL;
+    term->tempsblines = 0;
+    term->alt_sblines = 0;
     term->disptop = 0;
-    term->disptext = term->dispcurs = NULL;
+    term->disptext = NULL;
+    term->dispcursx = term->dispcursy = -1;
     term->tabs = NULL;
     deselect(term);
     term->rows = term->cols = -1;
     power_on(term);
     term->beephead = term->beeptail = NULL;
+#ifdef OPTIMISE_SCROLL
+    term->scrollhead = term->scrolltail = NULL;
+#endif /* OPTIMISE_SCROLL */
     term->nbeeps = 0;
     term->lastbeep = FALSE;
     term->beep_overloaded = FALSE;
     term->attr_mask = 0xffffffff;
     term->resize_fn = NULL;
     term->resize_ctx = NULL;
+    term->in_term_out = FALSE;
+    term->ltemp = NULL;
+    term->wcFrom = NULL;
+    term->wcTo = NULL;
+
+    term->bidi_cache_size = 0;
+    term->pre_bidi_cache = term->post_bidi_cache = NULL;
+
+    term->basic_erase_char.chr = CSET_ASCII | ' ';
+    term->basic_erase_char.attr = ATTR_DEFAULT;
 
     return term;
 }
 
 void term_free(Terminal *term)
 {
-    unsigned long *line;
+    termline *line;
     struct beeptime *beep;
+    int i;
 
     while ((line = delpos234(term->scrollback, 0)) != NULL)
-       sfree(line);
+       sfree(line);                   /* compressed data, not a termline */
     freetree234(term->scrollback);
     while ((line = delpos234(term->screen, 0)) != NULL)
-       sfree(line);
+       freeline(line);
     freetree234(term->screen);
     while ((line = delpos234(term->alt_screen, 0)) != NULL)
-       sfree(line);
+       freeline(line);
     freetree234(term->alt_screen);
+    if (term->disptext) {
+       for (i = 0; i < term->rows; i++)
+           freeline(term->disptext[i]);
+    }
     sfree(term->disptext);
     while (term->beephead) {
        beep = term->beephead;
@@ -416,6 +982,17 @@ void term_free(Terminal *term)
        printer_finish_job(term->print_job);
     bufchain_clear(&term->printer_buf);
     sfree(term->paste_buffer);
+    sfree(term->ltemp);
+    sfree(term->wcFrom);
+    sfree(term->wcTo);
+
+    for (i = 0; i < term->bidi_cache_size; i++) {
+       sfree(term->pre_bidi_cache[i]);
+       sfree(term->post_bidi_cache[i]);
+    }
+    sfree(term->pre_bidi_cache);
+    sfree(term->post_bidi_cache);
+
     sfree(term);
 }
 
@@ -425,7 +1002,7 @@ void term_free(Terminal *term)
 void term_size(Terminal *term, int newrows, int newcols, int newsavelines)
 {
     tree234 *newalt;
-    unsigned long *newdisp, *line;
+    termline **newdisp, *line;
     int i, j;
     int sblen;
     int save_alt_which = term->alt_which;
@@ -443,6 +1020,7 @@ void term_size(Terminal *term, int newrows, int newcols, int newsavelines)
     if (term->rows == -1) {
        term->scrollback = newtree234(NULL);
        term->screen = newtree234(NULL);
+       term->tempsblines = 0;
        term->rows = 0;
     }
 
@@ -452,15 +1030,14 @@ void term_size(Terminal *term, int newrows, int newcols, int newsavelines)
      * will take care of resizing each individual line if
      * necessary. So:
      * 
-     *  - If the new screen and the old screen differ in length, we
-     *    must shunt some lines in from the scrollback or out to
-     *    the scrollback.
-     * 
-     *  - If doing that fails to provide us with enough material to
-     *    fill the new screen (i.e. the number of rows needed in
-     *    the new screen exceeds the total number in the previous
-     *    screen+scrollback), we must invent some blank lines to
-     *    cover the gap.
+     *  - If the new screen is longer, we shunt lines in from temporary
+     *    scrollback if possible, otherwise we add new blank lines at
+     *    the bottom.
+     *
+     *  - If the new screen is shorter, we remove any blank lines at
+     *    the bottom if possible, otherwise shunt lines above the cursor
+     *    to scrollback if possible, otherwise delete lines below the
+     *    cursor.
      * 
      *  - Then, if the new scrollback length is less than the
      *    amount of scrollback we actually have, we must throw some
@@ -468,64 +1045,99 @@ void term_size(Terminal *term, int newrows, int newcols, int newsavelines)
      */
     sblen = count234(term->scrollback);
     /* Do this loop to expand the screen if newrows > rows */
-    for (i = term->rows; i < newrows; i++) {
-       if (sblen > 0) {
-           line = delpos234(term->scrollback, --sblen);
+    assert(term->rows == count234(term->screen));
+    while (term->rows < newrows) {
+       if (term->tempsblines > 0) {
+           unsigned char *cline;
+           /* Insert a line from the scrollback at the top of the screen. */
+           assert(sblen >= term->tempsblines);
+           cline = delpos234(term->scrollback, --sblen);
+           line = decompressline(cline, NULL);
+           sfree(cline);
+           line->temporary = FALSE;   /* reconstituted line is now real */
+           term->tempsblines -= 1;
+           addpos234(term->screen, line, 0);
+           term->curs.y += 1;
+           term->savecurs.y += 1;
        } else {
-           line = smalloc(TSIZE * (newcols + 2));
-           line[0] = newcols;
-           for (j = 0; j < newcols; j++)
-               line[j + 1] = ERASE_CHAR;
-            line[newcols] = LATTR_NORM;
+           /* Add a new blank line at the bottom of the screen. */
+           line = newline(term, newcols, FALSE);
+           addpos234(term->screen, line, count234(term->screen));
        }
-       addpos234(term->screen, line, 0);
+       term->rows += 1;
     }
     /* Do this loop to shrink the screen if newrows < rows */
-    for (i = newrows; i < term->rows; i++) {
-       line = delpos234(term->screen, 0);
-       addpos234(term->scrollback, line, sblen++);
+    while (term->rows > newrows) {
+       if (term->curs.y < term->rows - 1) {
+           /* delete bottom row, unless it contains the cursor */
+           sfree(delpos234(term->screen, term->rows - 1));
+       } else {
+           /* push top row to scrollback */
+           line = delpos234(term->screen, 0);
+           addpos234(term->scrollback, compressline(line), sblen++);
+           freeline(line);
+           term->tempsblines += 1;
+           term->curs.y -= 1;
+           term->savecurs.y -= 1;
+       }
+       term->rows -= 1;
     }
+    assert(term->rows == newrows);
     assert(count234(term->screen) == newrows);
+
+    /* Delete any excess lines from the scrollback. */
     while (sblen > newsavelines) {
        line = delpos234(term->scrollback, 0);
        sfree(line);
        sblen--;
     }
+    if (sblen < term->tempsblines)
+       term->tempsblines = sblen;
     assert(count234(term->scrollback) <= newsavelines);
+    assert(count234(term->scrollback) >= term->tempsblines);
     term->disptop = 0;
 
-    newdisp = smalloc(newrows * (newcols + 1) * TSIZE);
-    for (i = 0; i < newrows * (newcols + 1); i++)
-       newdisp[i] = ATTR_INVALID;
+    /* Make a new displayed text buffer. */
+    newdisp = snewn(newrows, termline *);
+    for (i = 0; i < newrows; i++) {
+       newdisp[i] = newline(term, newcols, FALSE);
+       for (j = 0; j < newcols; j++)
+           newdisp[i]->chars[i].attr = ATTR_INVALID;
+    }
+    if (term->disptext) {
+       for (i = 0; i < term->rows; i++)
+           freeline(term->disptext[i]);
+    }
     sfree(term->disptext);
     term->disptext = newdisp;
-    term->dispcurs = NULL;
+    term->dispcursx = term->dispcursy = -1;
 
+    /* Make a new alternate screen. */
     newalt = newtree234(NULL);
     for (i = 0; i < newrows; i++) {
-       line = smalloc(TSIZE * (newcols + 2));
-       line[0] = newcols;
-       for (j = 0; j < newcols; j++)
-           line[j + 1] = term->erase_char;
-        line[newcols + 1] = LATTR_NORM;
+       line = newline(term, newcols, TRUE);
        addpos234(newalt, line, i);
     }
     if (term->alt_screen) {
        while (NULL != (line = delpos234(term->alt_screen, 0)))
-           sfree(line);
+           freeline(line);
        freetree234(term->alt_screen);
     }
     term->alt_screen = newalt;
+    term->alt_sblines = 0;
 
-    term->tabs = srealloc(term->tabs, newcols * sizeof(*term->tabs));
+    term->tabs = sresize(term->tabs, newcols, unsigned char);
     {
        int i;
        for (i = (term->cols > 0 ? term->cols : 0); i < newcols; i++)
            term->tabs[i] = (i % 8 == 0 ? TRUE : FALSE);
     }
 
-    if (term->rows > 0)
-       term->curs.y += newrows - term->rows;
+    /* Check that the cursor positions are still valid. */
+    if (term->savecurs.y < 0)
+       term->savecurs.y = 0;
+    if (term->savecurs.y >= newrows)
+       term->savecurs.y = newrows - 1;
     if (term->curs.y < 0)
        term->curs.y = 0;
     if (term->curs.y >= newrows)
@@ -562,6 +1174,25 @@ void term_provide_resize_fn(Terminal *term,
        resize_fn(resize_ctx, term->cols, term->rows);
 }
 
+/* Find the bottom line on the screen that has any content.
+ * If only the top line has content, returns 0.
+ * If no lines have content, return -1.
+ */ 
+static int find_last_nonempty_line(Terminal * term, tree234 * screen)
+{
+    int i;
+    for (i = count234(screen) - 1; i >= 0; i--) {
+       termline *line = index234(screen, i);
+       int j;
+       for (j = 0; j < line->cols; j++) {
+           if (line->chars[j].chr != term->erase_char.chr ||
+               line->chars[j].attr != term->erase_char.attr) break;
+       }
+       if (j != line->cols) break;
+    }
+    return i;
+}
+
 /*
  * Swap screens. If `reset' is TRUE and we have been asked to
  * switch to the alternate screen, we must bring most of its
@@ -583,6 +1214,7 @@ static void swap_screen(Terminal *term, int which, int reset, int keep_cur_pos)
        ttr = term->alt_screen;
        term->alt_screen = term->screen;
        term->screen = ttr;
+       term->alt_sblines = find_last_nonempty_line(term, term->alt_screen) + 1;
        t = term->curs.x;
        if (!reset && !keep_cur_pos)
            term->curs.x = term->alt_x;
@@ -640,10 +1272,7 @@ static void swap_screen(Terminal *term, int which, int reset, int keep_cur_pos)
  */
 static void update_sbar(Terminal *term)
 {
-    int nscroll;
-
-    nscroll = count234(term->scrollback);
-
+    int nscroll = sblines(term);
     set_sbar(term->frontend, nscroll + term->rows,
             nscroll + term->disptop, term->rows);
 }
@@ -670,7 +1299,7 @@ static void check_selection(Terminal *term, pos from, pos to)
  */
 static void scroll(Terminal *term, int topline, int botline, int lines, int sb)
 {
-    unsigned long *line, *line2;
+    termline *line;
     int i, seltop, olddisptop, shift;
 
     if (topline != 0 || term->alt_which != 0)
@@ -681,23 +1310,23 @@ static void scroll(Terminal *term, int topline, int botline, int lines, int sb)
     if (lines < 0) {
        while (lines < 0) {
            line = delpos234(term->screen, botline);
-            line = resizeline(line, term->cols);
+            resizeline(term, line, term->cols);
            for (i = 0; i < term->cols; i++)
-               line[i + 1] = term->erase_char;
-           line[term->cols + 1] = 0;
+               line->chars[i] = term->erase_char;
+           line->lattr = LATTR_NORM;
            addpos234(term->screen, line, topline);
 
            if (term->selstart.y >= topline && term->selstart.y <= botline) {
                term->selstart.y++;
                if (term->selstart.y > botline) {
-                   term->selstart.y = botline;
+                   term->selstart.y = botline + 1;
                    term->selstart.x = 0;
                }
            }
            if (term->selend.y >= topline && term->selend.y <= botline) {
                term->selend.y++;
                if (term->selend.y > botline) {
-                   term->selend.y = botline;
+                   term->selend.y = botline + 1;
                    term->selend.x = 0;
                }
            }
@@ -711,18 +1340,21 @@ static void scroll(Terminal *term, int topline, int botline, int lines, int sb)
                int sblen = count234(term->scrollback);
                /*
                 * We must add this line to the scrollback. We'll
-                * remove a line from the top of the scrollback to
-                * replace it, or allocate a new one if the
-                * scrollback isn't full.
+                * remove a line from the top of the scrollback if
+                * the scrollback is full.
                 */
                if (sblen == term->savelines) {
-                   sblen--, line2 = delpos234(term->scrollback, 0);
-               } else {
-                   line2 = smalloc(TSIZE * (term->cols + 2));
-                   line2[0] = term->cols;
-               }
-               addpos234(term->scrollback, line, sblen);
-               line = line2;
+                   unsigned char *cline;
+
+                   sblen--;
+                   cline = delpos234(term->scrollback, 0);
+                   sfree(cline);
+               } else
+                   term->tempsblines += 1;
+
+               addpos234(term->scrollback, compressline(line), sblen);
+
+               line = newline(term, term->cols, TRUE);
 
                /*
                 * If the user is currently looking at part of the
@@ -741,10 +1373,10 @@ static void scroll(Terminal *term, int topline, int botline, int lines, int sb)
                if (term->disptop > -term->savelines && term->disptop < 0)
                    term->disptop--;
            }
-            line = resizeline(line, term->cols);
+            resizeline(term, line, term->cols);
            for (i = 0; i < term->cols; i++)
-               line[i + 1] = term->erase_char;
-           line[term->cols + 1] = LATTR_NORM;
+               line->chars[i] = term->erase_char;
+           line->lattr = LATTR_NORM;
            addpos234(term->screen, line, botline);
 
            /*
@@ -760,26 +1392,29 @@ static void scroll(Terminal *term, int topline, int botline, int lines, int sb)
             */
            seltop = sb ? -term->savelines : topline;
 
-           if (term->selstart.y >= seltop &&
-               term->selstart.y <= botline) {
-               term->selstart.y--;
-               if (term->selstart.y < seltop) {
-                   term->selstart.y = seltop;
-                   term->selstart.x = 0;
+           if (term->selstate != NO_SELECTION) {
+               if (term->selstart.y >= seltop &&
+                   term->selstart.y <= botline) {
+                   term->selstart.y--;
+                   if (term->selstart.y < seltop) {
+                       term->selstart.y = seltop;
+                       term->selstart.x = 0;
+                   }
                }
-           }
-           if (term->selend.y >= seltop && term->selend.y <= botline) {
-               term->selend.y--;
-               if (term->selend.y < seltop) {
-                   term->selend.y = seltop;
-                   term->selend.x = 0;
+               if (term->selend.y >= seltop && term->selend.y <= botline) {
+                   term->selend.y--;
+                   if (term->selend.y < seltop) {
+                       term->selend.y = seltop;
+                       term->selend.x = 0;
+                   }
                }
-           }
-           if (term->selanchor.y >= seltop && term->selanchor.y <= botline) {
-               term->selanchor.y--;
-               if (term->selanchor.y < seltop) {
-                   term->selanchor.y = seltop;
-                   term->selanchor.x = 0;
+               if (term->selanchor.y >= seltop &&
+                   term->selanchor.y <= botline) {
+                   term->selanchor.y--;
+                   if (term->selanchor.y < seltop) {
+                       term->selanchor.y = seltop;
+                       term->selanchor.x = 0;
+                   }
                }
            }
 
@@ -795,32 +1430,69 @@ static void scroll(Terminal *term, int topline, int botline, int lines, int sb)
 
 #ifdef OPTIMISE_SCROLL
 /*
+ * Add a scroll of a region on the screen into the pending scroll list.
+ * `lines' is +ve for scrolling forward, -ve for backward.
+ *
+ * If the scroll is on the same area as the last scroll in the list,
+ * merge them.
+ */
+static void save_scroll(Terminal *term, int topline, int botline, int lines)
+{
+    struct scrollregion *newscroll;
+    if (term->scrolltail &&
+       term->scrolltail->topline == topline && 
+       term->scrolltail->botline == botline) {
+       term->scrolltail->lines += lines;
+    } else {
+       newscroll = snew(struct scrollregion);
+       newscroll->topline = topline;
+       newscroll->botline = botline;
+       newscroll->lines = lines;
+       newscroll->next = NULL;
+
+       if (!term->scrollhead)
+           term->scrollhead = newscroll;
+       else
+           term->scrolltail->next = newscroll;
+       term->scrolltail = newscroll;
+    }
+}
+
+/*
  * Scroll the physical display, and our conception of it in disptext.
  */
 static void scroll_display(Terminal *term, int topline, int botline, int lines)
 {
-    unsigned long *start, *end;
-    int distance, size, i;
+    int distance, nlines, i;
 
-    start = term->disptext + topline * (term->cols + 1);
-    end = term->disptext + (botline + 1) * (term->cols + 1);
-    distance = (lines > 0 ? lines : -lines) * (term->cols + 1);
-    size = end - start - distance;
+    distance = lines > 0 ? lines : -lines;
+    nlines = botline - topline + 1 - distance;
     if (lines > 0) {
-       memmove(start, start + distance, size * TSIZE);
-       if (term->dispcurs >= start + distance &&
-           term->dispcurs <= start + distance + size)
-           term->dispcurs -= distance;
+       for (i = 0; i < nlines; i++)
+           memcpy(term->disptext[start+i]->chars,
+                  term->disptext[start+i+distance]->chars,
+                  term->cols * TSIZE);
+       if (term->dispcursy >= 0 &&
+           term->dispcursy >= topline + distance &&
+           term->dispcursy < topline + distance + nlines)
+           term->dispcursy -= distance;
        for (i = 0; i < distance; i++)
-           (start + size)[i] |= ATTR_INVALID;
+           for (j = 0; j < term->cols; j++)
+               term->disptext[start+nlines+i]->chars[j].attr |= ATTR_INVALID;
     } else {
-       memmove(start + distance, start, size * TSIZE);
-       if (term->dispcurs >= start && term->dispcurs <= start + size)
-           term->dispcurs += distance;
+       for (i = nlines; i-- ;)
+           memcpy(term->disptext[start+i+distance]->chars,
+                  term->disptext[start+i]->chars,
+                  term->cols * TSIZE);
+       if (term->dispcursy >= 0 &&
+           term->dispcursy >= topline &&
+           term->dispcursy < topline + nlines)
+           term->dispcursy += distance;
        for (i = 0; i < distance; i++)
-           start[i] |= ATTR_INVALID;
+           for (j = 0; j < term->cols; j++)
+               term->disptext[start+i]->chars[j].attr |= ATTR_INVALID;
     }
-    do_scroll(term->frontend, topline, botline, lines);
+    save_scroll(term, topline, botline, lines);
 }
 #endif /* OPTIMISE_SCROLL */
 
@@ -888,10 +1560,7 @@ static void save_cursor(Terminal *term, int save)
        term->cset_attr[term->cset] = term->save_csattr;
        term->sco_acs = term->save_sco_acs;
        fix_cpos;
-       if (term->use_bce)
-           term->erase_char = (' ' | ATTR_ASCII |
-                               (term->curr_attr &
-                                (ATTR_FGMASK | ATTR_BGMASK)));
+       set_erase_char(term);
     }
 }
 
@@ -916,19 +1585,19 @@ static void save_cursor(Terminal *term, int save)
  */
 static void check_boundary(Terminal *term, int x, int y)
 {
-    unsigned long *ldata;
+    termline *ldata;
 
     /* Validate input coordinates, just in case. */
     if (x == 0 || x > term->cols)
        return;
 
-    ldata = lineptr(y);
+    ldata = scrlineptr(y);
     if (x == term->cols) {
-       ldata[x] &= ~LATTR_WRAPPED2;
+       ldata->lattr &= ~LATTR_WRAPPED2;
     } else {
-       if ((ldata[x] & (CHAR_MASK | CSET_MASK)) == UCSWIDE) {
-           ldata[x-1] = ldata[x] =
-               (ldata[x-1] &~ (CHAR_MASK | CSET_MASK)) | ATTR_ASCII | ' ';
+       if (ldata->chars[x].chr == UCSWIDE) {
+           ldata->chars[x-1].chr = ' ' | CSET_ASCII;
+           ldata->chars[x] = ldata->chars[x-1];
        }
     }
 }
@@ -942,7 +1611,7 @@ static void erase_lots(Terminal *term,
 {
     pos start, end;
     int erase_lattr;
-    unsigned long *ldata;
+    int erasing_lines_from_top = 0;
 
     if (line_only) {
        start.y = term->curs.y;
@@ -972,17 +1641,45 @@ static void erase_lots(Terminal *term,
     if (start.y == 0 && start.x == 0 && end.y == term->rows)
        term_invalidate(term);
 
-    ldata = lineptr(start.y);
-    while (poslt(start, end)) {
-       if (start.x == term->cols) {
-            if (erase_lattr)
-                ldata[start.x] &= ~(LATTR_WRAPPED | LATTR_WRAPPED2);
-        } else {
-           ldata[start.x] = term->erase_char;
-        }
-       if (incpos(start) && start.y < term->rows)
-           ldata = lineptr(start.y);
+    /* Lines scrolled away shouldn't be brought back on if the terminal
+     * resizes. */
+    if (start.y == 0 && start.x == 0 && end.x == 0 && erase_lattr)
+       erasing_lines_from_top = 1;
+
+    if (term->cfg.erase_to_scrollback && erasing_lines_from_top) {
+       /* If it's a whole number of lines, starting at the top, and
+        * we're fully erasing them, erase by scrolling and keep the
+        * lines in the scrollback. */
+       int scrolllines = end.y;
+       if (end.y == term->rows) {
+           /* Shrink until we find a non-empty row.*/
+           scrolllines = find_last_nonempty_line(term, term->screen) + 1;
+       }
+       if (scrolllines > 0)
+           scroll(term, 0, scrolllines - 1, scrolllines, TRUE);
+       fix_cpos;
+    } else {
+       termline *ldata = scrlineptr(start.y);
+       while (poslt(start, end)) {
+           if (start.x == term->cols) {
+               if (!erase_lattr)
+                   ldata->lattr &= ~(LATTR_WRAPPED | LATTR_WRAPPED2);
+               else
+                   ldata->lattr = LATTR_NORM;
+           } else {
+               ldata->chars[start.x] = term->erase_char;
+           }
+           if (incpos(start) && start.y < term->rows) {
+               ldata = scrlineptr(start.y);
+           }
+       }
     }
+
+    /* After an erase of lines from the top of the screen, we shouldn't
+     * bring the lines back again if the terminal enlarges (since the user or
+     * application has explictly thrown them away). */
+    if (erasing_lines_from_top && !(term->alt_which))
+       term->tempsblines = 0;
 }
 
 /*
@@ -994,7 +1691,7 @@ static void insch(Terminal *term, int n)
     int dir = (n < 0 ? -1 : +1);
     int m;
     pos cursplus;
-    unsigned long *ldata;
+    termline *ldata;
 
     n = (n < 0 ? -n : n);
     if (n > term->cols - term->curs.x)
@@ -1006,15 +1703,17 @@ static void insch(Terminal *term, int n)
     check_boundary(term, term->curs.x, term->curs.y);
     if (dir < 0)
        check_boundary(term, term->curs.x + n, term->curs.y);
-    ldata = lineptr(term->curs.y);
+    ldata = scrlineptr(term->curs.y);
     if (dir < 0) {
-       memmove(ldata + term->curs.x, ldata + term->curs.x + n, m * TSIZE);
+       memmove(ldata->chars + term->curs.x, ldata->chars + term->curs.x + n,
+               m * TSIZE);
        while (n--)
-           ldata[term->curs.x + m++] = term->erase_char;
+           ldata->chars[term->curs.x + m++] = term->erase_char;
     } else {
-       memmove(ldata + term->curs.x + n, ldata + term->curs.x, m * TSIZE);
+       memmove(ldata->chars + term->curs.x + n, ldata->chars + term->curs.x,
+               m * TSIZE);
        while (n--)
-           ldata[term->curs.x + n] = term->erase_char;
+           ldata->chars[term->curs.x + n] = term->erase_char;
     }
 }
 
@@ -1028,10 +1727,10 @@ static void toggle_mode(Terminal *term, int mode, int query, int state)
 
     if (query)
        switch (mode) {
-         case 1:                      /* application cursor keys */
+         case 1:                      /* DECCKM: application cursor keys */
            term->app_cursor_keys = state;
            break;
-         case 2:                      /* VT52 mode */
+         case 2:                      /* DECANM: VT52 mode */
            term->vt52_mode = !state;
            if (term->vt52_mode) {
                term->blink_is_real = FALSE;
@@ -1040,13 +1739,17 @@ static void toggle_mode(Terminal *term, int mode, int query, int state)
                term->blink_is_real = term->cfg.blinktext;
            }
            break;
-         case 3:                      /* 80/132 columns */
+         case 3:                      /* DECCOLM: 80/132 columns */
            deselect(term);
            if (!term->cfg.no_remote_resize)
                request_resize(term->frontend, state ? 132 : 80, term->rows);
            term->reset_132 = state;
+           term->alt_t = term->marg_t = 0;
+           term->alt_b = term->marg_b = term->rows - 1;
+           move(term, 0, 0, 0);
+           erase_lots(term, FALSE, TRUE, TRUE);
            break;
-         case 5:                      /* reverse video */
+         case 5:                      /* DECSCNM: reverse video */
            /*
             * Toggle reverse video. If we receive an OFF within the
             * visual bell timeout period after an ON, we trigger an
@@ -1075,21 +1778,21 @@ static void toggle_mode(Terminal *term, int mode, int query, int state)
            if (state)
                term_update(term);
            break;
-         case 6:                      /* DEC origin mode */
+         case 6:                      /* DECOM: DEC origin mode */
            term->dec_om = state;
            break;
-         case 7:                      /* auto wrap */
+         case 7:                      /* DECAWM: auto wrap */
            term->wrap = state;
            break;
-         case 8:                      /* auto key repeat */
+         case 8:                      /* DECARM: auto key repeat */
            term->repeat_off = !state;
            break;
-         case 10:                     /* set local edit mode */
+         case 10:                     /* DECEDM: set local edit mode */
            term->term_editing = state;
            if (term->ldisc)           /* cause ldisc to notice changes */
                ldisc_send(term->ldisc, NULL, 0, 0);
            break;
-         case 25:                     /* enable/disable cursor */
+         case 25:                     /* DECTCEM: enable/disable cursor */
            compatibility2(OTHER, VT220);
            term->cursor_on = state;
            term->seen_disp_event = TRUE;
@@ -1115,35 +1818,36 @@ static void toggle_mode(Terminal *term, int mode, int query, int state)
            term->disptop = 0;
            break;
          case 1048:                   /* save/restore cursor */
-           save_cursor(term, state);
+           if (!term->cfg.no_alt_screen)
+                save_cursor(term, state);
            if (!state) term->seen_disp_event = TRUE;
            break;
          case 1049:                   /* cursor & alternate screen */
-           if (state)
+           if (state && !term->cfg.no_alt_screen)
                save_cursor(term, state);
            if (!state) term->seen_disp_event = TRUE;
            compatibility(OTHER);
            deselect(term);
            swap_screen(term, term->cfg.no_alt_screen ? 0 : state, TRUE, FALSE);
-           if (!state)
+           if (!state && !term->cfg.no_alt_screen)
                save_cursor(term, state);
            term->disptop = 0;
            break;
     } else
        switch (mode) {
-         case 4:                      /* set insert mode */
+         case 4:                      /* IRM: set insert mode */
            compatibility(VT102);
            term->insert = state;
            break;
-         case 12:                     /* set echo mode */
+         case 12:                     /* SRM: set echo mode */
            term->term_echoing = !state;
            if (term->ldisc)           /* cause ldisc to notice changes */
                ldisc_send(term->ldisc, NULL, 0, 0);
            break;
-         case 20:                     /* Return sends ... */
+         case 20:                     /* LNM: Return sends ... */
            term->cr_lf_return = state;
            break;
-         case 34:                     /* Make cursor BIG */
+         case 34:                     /* WYULCURM: Make cursor BIG */
            compatibility2(OTHER, VT220);
            term->big_cursor = !state;
        }
@@ -1231,7 +1935,8 @@ static void term_print_finish(Terminal *term)
  */
 void term_out(Terminal *term)
 {
-    int c, unget;
+    unsigned long c;
+    int unget;
     unsigned char localbuf[256], *chars;
     int nchars = 0;
 
@@ -1310,7 +2015,7 @@ void term_out(Terminal *term)
                        /* UTF-8 must be stateless so we ignore iso2022. */
                        if (term->ucsdata->unitab_ctrl[c] != 0xFF) 
                             c = term->ucsdata->unitab_ctrl[c];
-                       else c = ((unsigned char)c) | ATTR_ASCII;
+                       else c = ((unsigned char)c) | CSET_ASCII;
                        break;
                    } else if ((c & 0xe0) == 0xc0) {
                        term->utf_size = term->utf_state = 1;
@@ -1386,9 +2091,6 @@ void term_out(Terminal *term)
                    if (c == 0xFFFE || c == 0xFFFF)
                        c = UCSERR;
 
-                   /* Oops this is a 16bit implementation */
-                   if (c >= 0x10000)
-                       c = 0xFFFD;
                    break;
            }
            /* Are we in the nasty ACS mode? Note: no sco in utf mode. */
@@ -1396,7 +2098,7 @@ void term_out(Terminal *term)
                    (c!='\033' && c!='\012' && c!='\015' && c!='\b'))
            {
               if (term->sco_acs == 2) c |= 0x80;
-              c |= ATTR_SCOACS;
+              c |= CSET_SCOACS;
            } else {
                switch (term->cset_attr[term->cset]) {
                    /* 
@@ -1405,27 +2107,27 @@ void term_out(Terminal *term)
                     * range, make sure we use the same font as well as
                     * the same encoding.
                     */
-                 case ATTR_LINEDRW:
+                 case CSET_LINEDRW:
                    if (term->ucsdata->unitab_ctrl[c] != 0xFF)
                        c = term->ucsdata->unitab_ctrl[c];
                    else
-                       c = ((unsigned char) c) | ATTR_LINEDRW;
+                       c = ((unsigned char) c) | CSET_LINEDRW;
                    break;
 
-                 case ATTR_GBCHR:
+                 case CSET_GBCHR:
                    /* If UK-ASCII, make the '#' a LineDraw Pound */
                    if (c == '#') {
-                       c = '}' | ATTR_LINEDRW;
+                       c = '}' | CSET_LINEDRW;
                        break;
                    }
-                 /*FALLTHROUGH*/ case ATTR_ASCII:
+                 /*FALLTHROUGH*/ case CSET_ASCII:
                    if (term->ucsdata->unitab_ctrl[c] != 0xFF)
                        c = term->ucsdata->unitab_ctrl[c];
                    else
-                       c = ((unsigned char) c) | ATTR_ASCII;
+                       c = ((unsigned char) c) | CSET_ASCII;
                    break;
-               case ATTR_SCOACS:
-                   if (c>=' ') c = ((unsigned char)c) | ATTR_SCOACS;
+               case CSET_SCOACS:
+                   if (c>=' ') c = ((unsigned char)c) | CSET_SCOACS;
                    break;
                }
            }
@@ -1445,13 +2147,16 @@ void term_out(Terminal *term)
                term->curs.x--;
            term->wrapnext = FALSE;
            fix_cpos;
-           if (!term->cfg.no_dbackspace)    /* destructive bksp might be disabled */
-               *term->cpos = (' ' | term->curr_attr | ATTR_ASCII);
+           /* destructive backspace might be disabled */
+           if (!term->cfg.no_dbackspace) {
+               term->cpos->chr = ' ' | CSET_ASCII;
+               term->cpos->attr = term->curr_attr;
+           }
        } else
            /* Or normal C0 controls. */
-       if ((c & -32) == 0 && term->termstate < DO_CTRLS) {
+       if ((c & ~0x1F) == 0 && term->termstate < DO_CTRLS) {
            switch (c) {
-             case '\005':             /* terminal type query */
+             case '\005':             /* ENQ: terminal type query */
                /* Strictly speaking this is VT100 but a VT100 defaults to
                 * no response. Other terminals respond at their option.
                 *
@@ -1483,7 +2188,7 @@ void term_out(Terminal *term)
                               abuf, d - abuf, 0);
                }
                break;
-             case '\007':
+             case '\007':            /* BEL: Bell */
                {
                    struct beeptime *newbeep;
                    unsigned long ticks;
@@ -1491,7 +2196,7 @@ void term_out(Terminal *term)
                    ticks = GETTICKCOUNT();
 
                    if (!term->beep_overloaded) {
-                       newbeep = smalloc(sizeof(struct beeptime));
+                       newbeep = snew(struct beeptime);
                        newbeep->ticks = ticks;
                        newbeep->next = NULL;
                        if (!term->beephead)
@@ -1546,10 +2251,10 @@ void term_out(Terminal *term)
                            term_update(term);
                        }
                    }
-                   term->disptop = 0;
+                   term->seen_disp_event = TRUE;
                }
                break;
-             case '\b':
+             case '\b':              /* BS: Back space */
                if (term->curs.x == 0 &&
                    (term->curs.y == 0 || term->wrap == 0))
                    /* do nothing */ ;
@@ -1562,15 +2267,15 @@ void term_out(Terminal *term)
                fix_cpos;
                term->seen_disp_event = TRUE;
                break;
-             case '\016':
+             case '\016':            /* LS1: Locking-shift one */
                compatibility(VT100);
                term->cset = 1;
                break;
-             case '\017':
+             case '\017':            /* LS0: Locking-shift zero */
                compatibility(VT100);
                term->cset = 0;
                break;
-             case '\033':
+             case '\033':            /* ESC: Escape */
                if (term->vt52_mode)
                    term->termstate = VT52_ESC;
                else {
@@ -1579,7 +2284,7 @@ void term_out(Terminal *term)
                    term->esc_query = FALSE;
                }
                break;
-             case '\015':
+             case '\015':            /* CR: Carriage return */
                term->curs.x = 0;
                term->wrapnext = FALSE;
                fix_cpos;
@@ -1588,7 +2293,7 @@ void term_out(Terminal *term)
                if (term->logctx)
                    logtraffic(term->logctx, (unsigned char) c, LGTYP_ASCII);
                break;
-             case '\014':
+             case '\014':            /* FF: Form feed */
                if (has_compat(SCOANSI)) {
                    move(term, 0, 0, 0);
                    erase_lots(term, FALSE, FALSE, TRUE);
@@ -1597,9 +2302,9 @@ void term_out(Terminal *term)
                    term->seen_disp_event = 1;
                    break;
                }
-             case '\013':
+             case '\013':            /* VT: Line tabulation */
                compatibility(VT100);
-             case '\012':
+             case '\012':            /* LF: Line feed */
                if (term->curs.y == term->marg_b)
                    scroll(term, term->marg_t, term->marg_b, 1, TRUE);
                else if (term->curs.y < term->rows - 1)
@@ -1613,17 +2318,17 @@ void term_out(Terminal *term)
                if (term->logctx)
                    logtraffic(term->logctx, (unsigned char) c, LGTYP_ASCII);
                break;
-             case '\t':
+             case '\t':              /* HT: Character tabulation */
                {
                    pos old_curs = term->curs;
-                   unsigned long *ldata = lineptr(term->curs.y);
+                   termline *ldata = scrlineptr(term->curs.y);
 
                    do {
                        term->curs.x++;
                    } while (term->curs.x < term->cols - 1 &&
                             !term->tabs[term->curs.x]);
 
-                   if ((ldata[term->cols] & LATTR_MODE) != LATTR_NORM) {
+                   if ((ldata->lattr & LATTR_MODE) != LATTR_NORM) {
                        if (term->curs.x >= term->cols / 2)
                            term->curs.x = term->cols / 2 - 1;
                    } else {
@@ -1643,7 +2348,7 @@ void term_out(Terminal *term)
                /* Only graphic characters get this far;
                 * ctrls are stripped above */
                if (term->wrapnext && term->wrap) {
-                   term->cpos[1] |= LATTR_WRAPPED;
+                   scrlineptr(term->curs.y)->lattr |= LATTR_WRAPPED;
                    if (term->curs.y == term->marg_b)
                        scroll(term, term->marg_t, term->marg_b, 1, TRUE);
                    else if (term->curs.y < term->rows - 1)
@@ -1659,7 +2364,7 @@ void term_out(Terminal *term)
                    incpos(cursplus);
                    check_selection(term, term->curs, cursplus);
                }
-               if (((c & CSET_MASK) == ATTR_ASCII || (c & CSET_MASK) == 0) &&
+               if (((c & CSET_MASK) == CSET_ASCII || (c & CSET_MASK) == 0) &&
                    term->logctx)
                    logtraffic(term->logctx, (unsigned char) c, LGTYP_ASCII);
                {
@@ -1692,8 +2397,10 @@ void term_out(Terminal *term)
                        check_boundary(term, term->curs.x, term->curs.y);
                        check_boundary(term, term->curs.x+2, term->curs.y);
                        if (term->curs.x == term->cols-1) {
-                           *term->cpos++ = ATTR_ASCII | ' ' | term->curr_attr;
-                           *term->cpos |= LATTR_WRAPPED | LATTR_WRAPPED2;
+                           term->cpos->chr = CSET_ASCII | ' ';
+                           term->cpos->attr = term->curr_attr;
+                           scrlineptr(term->curs.y)->lattr |=
+                               LATTR_WRAPPED | LATTR_WRAPPED2;
                            if (term->curs.y == term->marg_b)
                                scroll(term, term->marg_t, term->marg_b,
                                       1, TRUE);
@@ -1705,14 +2412,19 @@ void term_out(Terminal *term)
                            check_boundary(term, term->curs.x, term->curs.y);
                            check_boundary(term, term->curs.x+2, term->curs.y);
                        }
-                       *term->cpos++ = c | term->curr_attr;
-                       *term->cpos++ = UCSWIDE | term->curr_attr;
+                       term->cpos->chr = c;
+                       term->cpos->attr = term->curr_attr;
+                       term->cpos++;
+                       term->cpos->chr = UCSWIDE;
+                       term->cpos->attr = term->curr_attr;
                        term->curs.x++;
                        break;
                      case 1:
                        check_boundary(term, term->curs.x, term->curs.y);
                        check_boundary(term, term->curs.x+1, term->curs.y);
-                       *term->cpos++ = c | term->curr_attr;
+                       term->cpos->chr = c;
+                       term->cpos->attr = term->curr_attr;
+                       term->cpos++;
                        break;
                      default:
                        continue;
@@ -1724,7 +2436,7 @@ void term_out(Terminal *term)
                    term->curs.x--;
                    term->wrapnext = TRUE;
                    if (term->wrap && term->vt52_mode) {
-                       term->cpos[1] |= LATTR_WRAPPED;
+                       scrlineptr(term->curs.y)->lattr |= LATTR_WRAPPED;
                        if (term->curs.y == term->marg_b)
                            scroll(term, term->marg_t, term->marg_b, 1, TRUE);
                        else if (term->curs.y < term->rows - 1)
@@ -1759,36 +2471,36 @@ void term_out(Terminal *term)
                }
                term->termstate = TOPLEVEL;
                switch (ANSI(c, term->esc_query)) {
-                 case '[':            /* enter CSI mode */
+                 case '[':             /* enter CSI mode */
                    term->termstate = SEEN_CSI;
                    term->esc_nargs = 1;
                    term->esc_args[0] = ARG_DEFAULT;
                    term->esc_query = FALSE;
                    break;
-                 case ']':            /* xterm escape sequences */
+                 case ']':             /* OSC: xterm escape sequences */
                    /* Compatibility is nasty here, xterm, linux, decterm yuk! */
                    compatibility(OTHER);
                    term->termstate = SEEN_OSC;
                    term->esc_args[0] = 0;
                    break;
-                 case '7':            /* save cursor */
+                 case '7':             /* DECSC: save cursor */
                    compatibility(VT100);
                    save_cursor(term, TRUE);
                    break;
-                 case '8':            /* restore cursor */
+                 case '8':             /* DECRC: restore cursor */
                    compatibility(VT100);
                    save_cursor(term, FALSE);
                    term->seen_disp_event = TRUE;
                    break;
-                 case '=':
+                 case '=':             /* DECKPAM: Keypad application mode */
                    compatibility(VT100);
                    term->app_keypad_keys = TRUE;
                    break;
-                 case '>':
+                 case '>':             /* DECKPNM: Keypad numeric mode */
                    compatibility(VT100);
                    term->app_keypad_keys = FALSE;
                    break;
-                 case 'D':            /* exactly equivalent to LF */
+                 case 'D':            /* IND: exactly equivalent to LF */
                    compatibility(VT100);
                    if (term->curs.y == term->marg_b)
                        scroll(term, term->marg_t, term->marg_b, 1, TRUE);
@@ -1798,7 +2510,7 @@ void term_out(Terminal *term)
                    term->wrapnext = FALSE;
                    term->seen_disp_event = TRUE;
                    break;
-                 case 'E':            /* exactly equivalent to CR-LF */
+                 case 'E':            /* NEL: exactly equivalent to CR-LF */
                    compatibility(VT100);
                    term->curs.x = 0;
                    if (term->curs.y == term->marg_b)
@@ -1809,7 +2521,7 @@ void term_out(Terminal *term)
                    term->wrapnext = FALSE;
                    term->seen_disp_event = TRUE;
                    break;
-                 case 'M':            /* reverse index - backwards LF */
+                 case 'M':            /* RI: reverse index - backwards LF */
                    compatibility(VT100);
                    if (term->curs.y == term->marg_t)
                        scroll(term, term->marg_t, term->marg_b, -1, TRUE);
@@ -1819,13 +2531,13 @@ void term_out(Terminal *term)
                    term->wrapnext = FALSE;
                    term->seen_disp_event = TRUE;
                    break;
-                 case 'Z':            /* terminal type query */
+                 case 'Z':            /* DECID: terminal type query */
                    compatibility(VT100);
                    if (term->ldisc)
                        ldisc_send(term->ldisc, term->id_string,
                                   strlen(term->id_string), 0);
                    break;
-                 case 'c':            /* restore power-on settings */
+                 case 'c':            /* RIS: restore power-on settings */
                    compatibility(VT100);
                    power_on(term);
                    if (term->ldisc)   /* cause ldisc to notice changes */
@@ -1839,23 +2551,25 @@ void term_out(Terminal *term)
                    term->disptop = 0;
                    term->seen_disp_event = TRUE;
                    break;
-                 case 'H':            /* set a tab */
+                 case 'H':            /* HTS: set a tab */
                    compatibility(VT100);
                    term->tabs[term->curs.x] = TRUE;
                    break;
 
-                 case ANSI('8', '#'):  /* ESC # 8 fills screen with Es :-) */
+                 case ANSI('8', '#'):  /* DECALN: fills screen with Es :-) */
                    compatibility(VT100);
                    {
-                       unsigned long *ldata;
+                       termline *ldata;
                        int i, j;
                        pos scrtop, scrbot;
 
                        for (i = 0; i < term->rows; i++) {
-                           ldata = lineptr(i);
-                           for (j = 0; j < term->cols; j++)
-                               ldata[j] = ATTR_DEFAULT | 'E';
-                           ldata[term->cols] = 0;
+                           ldata = scrlineptr(i);
+                           for (j = 0; j < term->cols; j++) {
+                               ldata->chars[j] = term->basic_erase_char;
+                               ldata->chars[j].chr = 'E';
+                           }
+                           ldata->lattr = LATTR_NORM;
                        }
                        term->disptop = 0;
                        term->seen_disp_event = TRUE;
@@ -1872,70 +2586,68 @@ void term_out(Terminal *term)
                  case ANSI('6', '#'):
                    compatibility(VT100);
                    {
-                       unsigned long nlattr;
-                       unsigned long *ldata;
+                       int nlattr;
+
                        switch (ANSI(c, term->esc_query)) {
-                         case ANSI('3', '#'):
+                         case ANSI('3', '#'): /* DECDHL: 2*height, top */
                            nlattr = LATTR_TOP;
                            break;
-                         case ANSI('4', '#'):
+                         case ANSI('4', '#'): /* DECDHL: 2*height, bottom */
                            nlattr = LATTR_BOT;
                            break;
-                         case ANSI('5', '#'):
+                         case ANSI('5', '#'): /* DECSWL: normal */
                            nlattr = LATTR_NORM;
                            break;
-                         default: /* spiritually case ANSI('6', '#'): */
+                         default: /* case ANSI('6', '#'): DECDWL: 2*width */
                            nlattr = LATTR_WIDE;
                            break;
                        }
-                       ldata = lineptr(term->curs.y);
-                       ldata[term->cols] &= ~LATTR_MODE;
-                       ldata[term->cols] |= nlattr;
+                       scrlineptr(term->curs.y)->lattr = nlattr;
                    }
                    break;
-
+                 /* GZD4: G0 designate 94-set */
                  case ANSI('A', '('):
                    compatibility(VT100);
                    if (!term->cfg.no_remote_charset)
-                       term->cset_attr[0] = ATTR_GBCHR;
+                       term->cset_attr[0] = CSET_GBCHR;
                    break;
                  case ANSI('B', '('):
                    compatibility(VT100);
                    if (!term->cfg.no_remote_charset)
-                       term->cset_attr[0] = ATTR_ASCII;
+                       term->cset_attr[0] = CSET_ASCII;
                    break;
                  case ANSI('0', '('):
                    compatibility(VT100);
                    if (!term->cfg.no_remote_charset)
-                       term->cset_attr[0] = ATTR_LINEDRW;
+                       term->cset_attr[0] = CSET_LINEDRW;
                    break;
                  case ANSI('U', '('): 
                    compatibility(OTHER);
                    if (!term->cfg.no_remote_charset)
-                       term->cset_attr[0] = ATTR_SCOACS; 
+                       term->cset_attr[0] = CSET_SCOACS; 
                    break;
-
+                 /* G1D4: G1-designate 94-set */
                  case ANSI('A', ')'):
                    compatibility(VT100);
                    if (!term->cfg.no_remote_charset)
-                       term->cset_attr[1] = ATTR_GBCHR;
+                       term->cset_attr[1] = CSET_GBCHR;
                    break;
                  case ANSI('B', ')'):
                    compatibility(VT100);
                    if (!term->cfg.no_remote_charset)
-                       term->cset_attr[1] = ATTR_ASCII;
+                       term->cset_attr[1] = CSET_ASCII;
                    break;
                  case ANSI('0', ')'):
                    compatibility(VT100);
                    if (!term->cfg.no_remote_charset)
-                       term->cset_attr[1] = ATTR_LINEDRW;
+                       term->cset_attr[1] = CSET_LINEDRW;
                    break;
                  case ANSI('U', ')'): 
                    compatibility(OTHER);
                    if (!term->cfg.no_remote_charset)
-                       term->cset_attr[1] = ATTR_SCOACS; 
+                       term->cset_attr[1] = CSET_SCOACS; 
                    break;
-
+                 /* DOCS: Designate other coding system */
                  case ANSI('8', '%'):  /* Old Linux code */
                  case ANSI('G', '%'):
                    compatibility(OTHER);
@@ -1973,59 +2685,59 @@ void term_out(Terminal *term)
                    term->termstate = SEEN_CSI;
                } else
                    switch (ANSI(c, term->esc_query)) {
-                     case 'A':       /* move up N lines */
+                     case 'A':       /* CUU: move up N lines */
                        move(term, term->curs.x,
                             term->curs.y - def(term->esc_args[0], 1), 1);
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'e':       /* move down N lines */
+                     case 'e':         /* VPR: move down N lines */
                        compatibility(ANSI);
                        /* FALLTHROUGH */
-                     case 'B':
+                     case 'B':         /* CUD: Cursor down */
                        move(term, term->curs.x,
                             term->curs.y + def(term->esc_args[0], 1), 1);
                        term->seen_disp_event = TRUE;
                        break;
-                     case ANSI('c', '>'):      /* report xterm version */
+                     case ANSI('c', '>'):      /* DA: report xterm version */
                        compatibility(OTHER);
                        /* this reports xterm version 136 so that VIM can
                           use the drag messages from the mouse reporting */
                        if (term->ldisc)
                            ldisc_send(term->ldisc, "\033[>0;136;0c", 11, 0);
                        break;
-                     case 'a':       /* move right N cols */
+                     case 'a':         /* HPR: move right N cols */
                        compatibility(ANSI);
                        /* FALLTHROUGH */
-                     case 'C':
+                     case 'C':         /* CUF: Cursor right */ 
                        move(term, term->curs.x + def(term->esc_args[0], 1),
                             term->curs.y, 1);
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'D':       /* move left N cols */
+                     case 'D':       /* CUB: move left N cols */
                        move(term, term->curs.x - def(term->esc_args[0], 1),
                             term->curs.y, 1);
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'E':       /* move down N lines and CR */
+                     case 'E':       /* CNL: move down N lines and CR */
                        compatibility(ANSI);
                        move(term, 0,
                             term->curs.y + def(term->esc_args[0], 1), 1);
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'F':       /* move up N lines and CR */
+                     case 'F':       /* CPL: move up N lines and CR */
                        compatibility(ANSI);
                        move(term, 0,
                             term->curs.y - def(term->esc_args[0], 1), 1);
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'G':
-                     case '`':       /* set horizontal posn */
+                     case 'G':       /* CHA */
+                     case '`':       /* HPA: set horizontal posn */
                        compatibility(ANSI);
                        move(term, def(term->esc_args[0], 1) - 1,
                             term->curs.y, 0);
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'd':       /* set vertical posn */
+                     case 'd':       /* VPA: set vertical posn */
                        compatibility(ANSI);
                        move(term, term->curs.x,
                             ((term->dec_om ? term->marg_t : 0) +
@@ -2033,8 +2745,8 @@ void term_out(Terminal *term)
                             (term->dec_om ? 2 : 0));
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'H':
-                     case 'f':       /* set horz and vert posns at once */
+                     case 'H':      /* CUP */
+                     case 'f':      /* HVP: set horz and vert posns at once */
                        if (term->esc_nargs < 2)
                            term->esc_args[1] = ARG_DEFAULT;
                        move(term, def(term->esc_args[1], 1) - 1,
@@ -2043,7 +2755,7 @@ void term_out(Terminal *term)
                             (term->dec_om ? 2 : 0));
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'J':       /* erase screen or parts of it */
+                     case 'J':       /* ED: erase screen or parts of it */
                        {
                            unsigned int i = def(term->esc_args[0], 0) + 1;
                            if (i > 3)
@@ -2053,7 +2765,7 @@ void term_out(Terminal *term)
                        term->disptop = 0;
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'K':       /* erase line or parts of it */
+                     case 'K':       /* EL: erase line or parts of it */
                        {
                            unsigned int i = def(term->esc_args[0], 0) + 1;
                            if (i > 3)
@@ -2062,7 +2774,7 @@ void term_out(Terminal *term)
                        }
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'L':       /* insert lines */
+                     case 'L':       /* IL: insert lines */
                        compatibility(VT102);
                        if (term->curs.y <= term->marg_b)
                            scroll(term, term->curs.y, term->marg_b,
@@ -2070,7 +2782,7 @@ void term_out(Terminal *term)
                        fix_cpos;
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'M':       /* delete lines */
+                     case 'M':       /* DL: delete lines */
                        compatibility(VT102);
                        if (term->curs.y <= term->marg_b)
                            scroll(term, term->curs.y, term->marg_b,
@@ -2079,25 +2791,25 @@ void term_out(Terminal *term)
                        fix_cpos;
                        term->seen_disp_event = TRUE;
                        break;
-                     case '@':       /* insert chars */
+                     case '@':       /* ICH: insert chars */
                        /* XXX VTTEST says this is vt220, vt510 manual says vt102 */
                        compatibility(VT102);
                        insch(term, def(term->esc_args[0], 1));
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'P':       /* delete chars */
+                     case 'P':       /* DCH: delete chars */
                        compatibility(VT102);
                        insch(term, -def(term->esc_args[0], 1));
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'c':       /* terminal type query */
+                     case 'c':       /* DA: terminal type query */
                        compatibility(VT100);
                        /* This is the response for a VT102 */
                        if (term->ldisc)
                            ldisc_send(term->ldisc, term->id_string,
                                       strlen(term->id_string), 0);
                        break;
-                     case 'n':       /* cursor position query */
+                     case 'n':       /* DSR: cursor position query */
                        if (term->ldisc) {
                            if (term->esc_args[0] == 6) {
                                char buf[32];
@@ -2109,7 +2821,7 @@ void term_out(Terminal *term)
                            }
                        }
                        break;
-                     case 'h':       /* toggle modes to high */
+                     case 'h':       /* SM: toggle modes to high */
                      case ANSI_QUE('h'):
                        compatibility(VT100);
                        {
@@ -2119,7 +2831,7 @@ void term_out(Terminal *term)
                                            term->esc_query, TRUE);
                        }
                        break;
-                     case 'i':
+                     case 'i':         /* MC: Media copy */
                      case ANSI_QUE('i'):
                        compatibility(VT100);
                        {
@@ -2135,7 +2847,7 @@ void term_out(Terminal *term)
                            }
                        }
                        break;                  
-                     case 'l':       /* toggle modes to low */
+                     case 'l':       /* RM: toggle modes to low */
                      case ANSI_QUE('l'):
                        compatibility(VT100);
                        {
@@ -2145,7 +2857,7 @@ void term_out(Terminal *term)
                                            term->esc_query, FALSE);
                        }
                        break;
-                     case 'g':       /* clear tabs */
+                     case 'g':       /* TBC: clear tabs */
                        compatibility(VT100);
                        if (term->esc_nargs == 1) {
                            if (term->esc_args[0] == 0) {
@@ -2157,7 +2869,7 @@ void term_out(Terminal *term)
                            }
                        }
                        break;
-                     case 'r':       /* set scroll margins */
+                     case 'r':       /* DECSTBM: set scroll margins */
                        compatibility(VT100);
                        if (term->esc_nargs <= 2) {
                            int top, bot;
@@ -2191,7 +2903,7 @@ void term_out(Terminal *term)
                            }
                        }
                        break;
-                     case 'm':       /* set graphics rendition */
+                     case 'm':       /* SGR: set graphics rendition */
                        {
                            /* 
                             * A VT100 without the AVO only had one
@@ -2225,7 +2937,7 @@ void term_out(Terminal *term)
                            for (i = 0; i < term->esc_nargs; i++) {
                                switch (def(term->esc_args[i], 0)) {
                                  case 0:       /* restore defaults */
-                                   term->curr_attr = ATTR_DEFAULT;
+                                   term->curr_attr = term->default_attr;
                                    break;
                                  case 1:       /* enable bold */
                                    compatibility(VT100AVO);
@@ -2241,6 +2953,11 @@ void term_out(Terminal *term)
                                    compatibility(VT100AVO);
                                    term->curr_attr |= ATTR_BLINK;
                                    break;
+                                 case 6:       /* SCO light bkgrd */
+                                   compatibility(SCOANSI);
+                                   term->blink_is_real = FALSE;
+                                   term->curr_attr |= ATTR_BLINK;
+                                   break;
                                  case 7:       /* enable reverse video */
                                    term->curr_attr |= ATTR_REVERSE;
                                    break;
@@ -2336,11 +3053,7 @@ void term_out(Terminal *term)
                                    break;
                                }
                            }
-                           if (term->use_bce)
-                               term->erase_char = (' ' | ATTR_ASCII |
-                                                   (term->curr_attr & 
-                                                    (ATTR_FGMASK |
-                                                     ATTR_BGMASK)));
+                           set_erase_char(term);
                        }
                        break;
                      case 's':       /* save cursor */
@@ -2350,7 +3063,7 @@ void term_out(Terminal *term)
                        save_cursor(term, FALSE);
                        term->seen_disp_event = TRUE;
                        break;
-                     case 't':       /* set page size - ie window height */
+                     case 't': /* DECSLPP: set page size - ie window height */
                        /*
                         * VT340/VT420 sequence DECSLPP, DEC only allows values
                         *  24/25/36/48/72/144 other emulators (eg dtterm) use
@@ -2463,7 +3176,8 @@ void term_out(Terminal *term)
                                 */
                                break;
                              case 20:
-                               if (term->ldisc) {
+                               if (term->ldisc &&
+                                   !term->cfg.no_remote_qtitle) {
                                    p = get_window_title(term->frontend, TRUE);
                                    len = strlen(p);
                                    ldisc_send(term->ldisc, "\033]L", 3, 0);
@@ -2472,7 +3186,8 @@ void term_out(Terminal *term)
                                }
                                break;
                              case 21:
-                               if (term->ldisc) {
+                               if (term->ldisc &&
+                                   !term->cfg.no_remote_qtitle) {
                                    p = get_window_title(term->frontend,FALSE);
                                    len = strlen(p);
                                    ldisc_send(term->ldisc, "\033]l", 3, 0);
@@ -2483,7 +3198,7 @@ void term_out(Terminal *term)
                            }
                        }
                        break;
-                     case 'S':
+                     case 'S':         /* SU: Scroll up */
                        compatibility(SCOANSI);
                        scroll(term, term->marg_t, term->marg_b,
                               def(term->esc_args[0], 1), TRUE);
@@ -2491,7 +3206,7 @@ void term_out(Terminal *term)
                        term->wrapnext = FALSE;
                        term->seen_disp_event = TRUE;
                        break;
-                     case 'T':
+                     case 'T':         /* SD: Scroll down */
                        compatibility(SCOANSI);
                        scroll(term, term->marg_t, term->marg_b,
                               -def(term->esc_args[0], 1), TRUE);
@@ -2499,11 +3214,12 @@ void term_out(Terminal *term)
                        term->wrapnext = FALSE;
                        term->seen_disp_event = TRUE;
                        break;
-                     case ANSI('|', '*'):
-                       /* VT420 sequence DECSNLS
+                     case ANSI('|', '*'): /* DECSNLS */
+                       /* 
                         * Set number of lines on screen
-                        * VT420 uses VGA like hardware and can support any size in
-                        * reasonable range (24..49 AIUI) with no default specified.
+                        * VT420 uses VGA like hardware and can
+                        * support any size in reasonable range
+                        * (24..49 AIUI) with no default specified.
                         */
                        compatibility(VT420);
                        if (term->esc_nargs == 1 && term->esc_args[0] > 0) {
@@ -2514,10 +3230,11 @@ void term_out(Terminal *term)
                            deselect(term);
                        }
                        break;
-                     case ANSI('|', '$'):
-                       /* VT340/VT420 sequence DECSCPP
+                     case ANSI('|', '$'): /* DECSCPP */
+                       /*
                         * Set number of columns per page
-                        * Docs imply range is only 80 or 132, but I'll allow any.
+                        * Docs imply range is only 80 or 132, but
+                        * I'll allow any.
                         */
                        compatibility(VT340TEXT);
                        if (term->esc_nargs <= 1) {
@@ -2528,13 +3245,14 @@ void term_out(Terminal *term)
                            deselect(term);
                        }
                        break;
-                     case 'X':       /* write N spaces w/o moving cursor */
-                       /* XXX VTTEST says this is vt220, vt510 manual says vt100 */
+                     case 'X':     /* ECH: write N spaces w/o moving cursor */
+                       /* XXX VTTEST says this is vt220, vt510 manual
+                        * says vt100 */
                        compatibility(ANSIMIN);
                        {
                            int n = def(term->esc_args[0], 1);
                            pos cursplus;
-                           unsigned long *p = term->cpos;
+                           termchar *p = term->cpos;
                            if (n > term->cols - term->curs.x)
                                n = term->cols - term->curs.x;
                            cursplus = term->curs;
@@ -2547,7 +3265,7 @@ void term_out(Terminal *term)
                            term->seen_disp_event = TRUE;
                        }
                        break;
-                     case 'x':       /* report terminal characteristics */
+                     case 'x':       /* DECREQTPARM: report terminal characteristics */
                        compatibility(VT100);
                        if (term->ldisc) {
                            char buf[32];
@@ -2559,7 +3277,7 @@ void term_out(Terminal *term)
                            }
                        }
                        break;
-                     case 'Z':         /* BackTab for xterm */
+                     case 'Z':         /* CBT: BackTab for xterm */
                        compatibility(OTHER);
                        {
                            int i = def(term->esc_args[0], 1);
@@ -2575,20 +3293,81 @@ void term_out(Terminal *term)
                            check_selection(term, old_curs, term->curs);
                        }
                        break;
-                     case ANSI('L', '='):
-                       compatibility(OTHER);
-                       term->use_bce = (term->esc_args[0] <= 0);
-                       term->erase_char = ERASE_CHAR;
-                       if (term->use_bce)
-                           term->erase_char = (' ' | ATTR_ASCII |
-                                               (term->curr_attr & 
-                                                (ATTR_FGMASK | ATTR_BGMASK)));
+                     case ANSI('c', '='):      /* Hide or Show Cursor */
+                       compatibility(SCOANSI);
+                       switch(term->esc_args[0]) {
+                         case 0:  /* hide cursor */
+                           term->cursor_on = FALSE;
+                           break;
+                         case 1:  /* restore cursor */
+                           term->big_cursor = FALSE;
+                           term->cursor_on = TRUE;
+                           break;
+                         case 2:  /* block cursor */
+                           term->big_cursor = TRUE;
+                           term->cursor_on = TRUE;
+                           break;
+                       }
+                       break;
+                     case ANSI('C', '='):
+                       /*
+                        * set cursor start on scanline esc_args[0] and
+                        * end on scanline esc_args[1].If you set
+                        * the bottom scan line to a value less than
+                        * the top scan line, the cursor will disappear.
+                        */
+                       compatibility(SCOANSI);
+                       if (term->esc_nargs >= 2) {
+                           if (term->esc_args[0] > term->esc_args[1])
+                               term->cursor_on = FALSE;
+                           else
+                               term->cursor_on = TRUE;
+                       }
+                       break;
+                     case ANSI('D', '='):
+                       compatibility(SCOANSI);
+                       term->blink_is_real = FALSE;
+                       if (term->esc_args[0]>=1)
+                           term->curr_attr |= ATTR_BLINK;
+                       else
+                           term->curr_attr &= ~ATTR_BLINK;
                        break;
                      case ANSI('E', '='):
-                       compatibility(OTHER);
+                       compatibility(SCOANSI);
                        term->blink_is_real = (term->esc_args[0] >= 1);
                        break;
-                     case ANSI('p', '"'):
+                     case ANSI('F', '='):      /* set normal foreground */
+                       compatibility(SCOANSI);
+                       if (term->esc_args[0] >= 0 && term->esc_args[0] < 16) {
+                           long colour =
+                               (sco2ansicolour[term->esc_args[0] & 0x7] |
+                                ((term->esc_args[0] & 0x8) << 1)) <<
+                               ATTR_FGSHIFT;
+                           term->curr_attr &= ~ATTR_FGMASK;
+                           term->curr_attr |= colour;
+                           term->default_attr &= ~ATTR_FGMASK;
+                           term->default_attr |= colour;
+                       }
+                       break;
+                     case ANSI('G', '='):      /* set normal background */
+                       compatibility(SCOANSI);
+                       if (term->esc_args[0] >= 0 && term->esc_args[0] < 16) {
+                           long colour =
+                               (sco2ansicolour[term->esc_args[0] & 0x7] |
+                                ((term->esc_args[0] & 0x8) << 1)) <<
+                               ATTR_BGSHIFT;
+                           term->curr_attr &= ~ATTR_BGMASK;
+                           term->curr_attr |= colour;
+                           term->default_attr &= ~ATTR_BGMASK;
+                           term->default_attr |= colour;
+                       }
+                       break;
+                     case ANSI('L', '='):
+                       compatibility(SCOANSI);
+                       term->use_bce = (term->esc_args[0] <= 0);
+                       set_erase_char(term);
+                       break;
+                     case ANSI('p', '"'): /* DECSCL: set compat level */
                        /*
                         * Allow the host to make this emulator a
                         * 'perfect' VT102. This first appeared in
@@ -2738,17 +3517,17 @@ void term_out(Terminal *term)
                } else if (c == '\033')
                    term->termstate = OSC_MAYBE_ST;
                else if (term->osc_strlen < OSC_STR_MAX)
-                   term->osc_string[term->osc_strlen++] = c;
+                   term->osc_string[term->osc_strlen++] = (char)c;
                break;
              case SEEN_OSC_P:
                {
                    int max = (term->osc_strlen == 0 ? 21 : 16);
                    int val;
-                   if (c >= '0' && c <= '9')
+                   if ((int)c >= '0' && (int)c <= '9')
                        val = c - '0';
-                   else if (c >= 'A' && c <= 'A' + max - 10)
+                   else if ((int)c >= 'A' && (int)c <= 'A' + max - 10)
                        val = c - 'A' + 10;
-                   else if (c >= 'a' && c <= 'a' + max - 10)
+                   else if ((int)c >= 'a' && (int)c <= 'a' + max - 10)
                        val = c - 'a' + 10;
                    else {
                        term->termstate = TOPLEVEL;
@@ -2841,10 +3620,10 @@ void term_out(Terminal *term)
                     *
                     */
                  case 'F':
-                   term->cset_attr[term->cset = 0] = ATTR_LINEDRW;
+                   term->cset_attr[term->cset = 0] = CSET_LINEDRW;
                    break;
                  case 'G':
-                   term->cset_attr[term->cset = 0] = ATTR_ASCII;
+                   term->cset_attr[term->cset = 0] = CSET_ASCII;
                    break;
                  case 'H':
                    move(term, 0, 0, 0);
@@ -2980,10 +3759,7 @@ void term_out(Terminal *term)
                    /* compatibility(OTHER) */
                    term->vt52_bold = FALSE;
                    term->curr_attr = ATTR_DEFAULT;
-                   if (term->use_bce)
-                       term->erase_char = (' ' | ATTR_ASCII |
-                                           (term->curr_attr & 
-                                            (ATTR_FGMASK | ATTR_BGMASK)));
+                   set_erase_char(term);
                    break;
                  case 'S':
                    /* compatibility(VI50) */
@@ -3024,10 +3800,7 @@ void term_out(Terminal *term)
                if ((c & 0x8) || term->vt52_bold)
                    term->curr_attr |= ATTR_BOLD;
 
-               if (term->use_bce)
-                   term->erase_char = (' ' | ATTR_ASCII |
-                                       (term->curr_attr &
-                                        (ATTR_FGMASK | ATTR_BGMASK)));
+               set_erase_char(term);
                break;
              case VT52_BG:
                term->termstate = TOPLEVEL;
@@ -3039,10 +3812,7 @@ void term_out(Terminal *term)
                if (c & 0x8)
                    term->curr_attr |= ATTR_BLINK;
 
-               if (term->use_bce)
-                   term->erase_char = (' ' | ATTR_ASCII |
-                                       (term->curr_attr &
-                                        (ATTR_FGMASK | ATTR_BGMASK)));
+               set_erase_char(term);
                break;
 #endif
              default: break;          /* placate gcc warning about enum use */
@@ -3055,23 +3825,59 @@ void term_out(Terminal *term)
     }
 
     term_print_flush(term);
+    logflush(term->logctx);
 }
 
-#if 0
 /*
- * Compare two lines to determine whether they are sufficiently
- * alike to scroll-optimise one to the other. Return the degree of
- * similarity.
+ * To prevent having to run the reasonably tricky bidi algorithm
+ * too many times, we maintain a cache of the last lineful of data
+ * fed to the algorithm on each line of the display.
  */
-static int linecmp(Terminal *term, unsigned long *a, unsigned long *b)
+static int term_bidi_cache_hit(Terminal *term, int line,
+                              termchar *lbefore, int width)
+{
+    if (!term->pre_bidi_cache)
+       return FALSE;                  /* cache doesn't even exist yet! */
+
+    if (line >= term->bidi_cache_size)
+       return FALSE;                  /* cache doesn't have this many lines */
+
+    if (!term->pre_bidi_cache[line])
+       return FALSE;                  /* cache doesn't contain _this_ line */
+
+    if (!memcmp(term->pre_bidi_cache[line], lbefore, width * TSIZE))
+       return TRUE;                   /* aha! the line matches the cache */
+
+    return FALSE;                     /* it didn't match. */
+}
+
+static void term_bidi_cache_store(Terminal *term, int line, termchar *lbefore,
+                                 termchar *lafter, int width)
 {
-    int i, n;
+    if (!term->pre_bidi_cache || term->bidi_cache_size <= line) {
+       int j = term->bidi_cache_size;
+       term->bidi_cache_size = line+1;
+       term->pre_bidi_cache = sresize(term->pre_bidi_cache,
+                                      term->bidi_cache_size,
+                                      termchar *);
+       term->post_bidi_cache = sresize(term->post_bidi_cache,
+                                       term->bidi_cache_size,
+                                       termchar *);
+       while (j < term->bidi_cache_size) {
+           term->pre_bidi_cache[j] = term->post_bidi_cache[j] = NULL;
+           j++;
+       }
+    }
+
+    sfree(term->pre_bidi_cache[line]);
+    sfree(term->post_bidi_cache[line]);
 
-    for (i = n = 0; i < term->cols; i++)
-       n += (*a++ == *b++);
-    return n;
+    term->pre_bidi_cache[line] = snewn(width, termchar);
+    term->post_bidi_cache[line] = snewn(width, termchar);
+
+    memcpy(term->pre_bidi_cache[line], lbefore, width * TSIZE);
+    memcpy(term->post_bidi_cache[line], lafter, width * TSIZE);
 }
-#endif
 
 /*
  * Given a context, update the window. Out of paranoia, we don't
@@ -3079,12 +3885,17 @@ static int linecmp(Terminal *term, unsigned long *a, unsigned long *b)
  */
 static void do_paint(Terminal *term, Context ctx, int may_optimise)
 {
-    int i, j, our_curs_y, our_curs_x;
-    unsigned long rv, cursor;
+    int i, it, j, our_curs_y, our_curs_x;
+    int rv, cursor;
     pos scrpos;
-    char ch[1024];
-    long cursor_background = ERASE_CHAR;
+    wchar_t ch[1024];
+    termchar cursor_background;
     unsigned long ticks;
+#ifdef OPTIMISE_SCROLL
+    struct scrollregion *sr;
+#endif /* OPTIMISE_SCROLL */
+
+    cursor_background = term->basic_erase_char;
 
     /*
      * Check the visual bell state.
@@ -3092,8 +3903,8 @@ static void do_paint(Terminal *term, Context ctx, int may_optimise)
     if (term->in_vbell) {
        ticks = GETTICKCOUNT();
        if (ticks - term->vbell_startpoint >= VBELL_TIMEOUT)
-           term->in_vbell = FALSE; 
-   }
+           term->in_vbell = FALSE;
+    }
 
     rv = (!term->rvideo ^ !term->in_vbell ? ATTR_REVERSE : 0);
 
@@ -3126,33 +3937,54 @@ static void do_paint(Terminal *term, Context ctx, int may_optimise)
         * to display the cursor covering the _whole_ character,
         * exactly as if it were one space to the left.
         */
-       unsigned long *ldata = lineptr(term->curs.y);
+       termline *ldata = lineptr(term->curs.y);
        our_curs_x = term->curs.x;
        if (our_curs_x > 0 &&
-           (ldata[our_curs_x] & (CHAR_MASK | CSET_MASK)) == UCSWIDE)
+           ldata->chars[our_curs_x].chr == UCSWIDE)
            our_curs_x--;
+       unlineptr(ldata);
     }
 
-    if (term->dispcurs && (term->curstype != cursor ||
-                          term->dispcurs !=
-                          term->disptext + our_curs_y * (term->cols + 1) +
-                          our_curs_x)) {
-       if (term->dispcurs > term->disptext && 
-           (*term->dispcurs & (CHAR_MASK | CSET_MASK)) == UCSWIDE)
-           term->dispcurs[-1] |= ATTR_INVALID;
-       if ( (term->dispcurs[1] & (CHAR_MASK | CSET_MASK)) == UCSWIDE)
-           term->dispcurs[1] |= ATTR_INVALID;
-       *term->dispcurs |= ATTR_INVALID;
+    /*
+     * If the cursor is not where it was last time we painted, and
+     * its previous position is visible on screen, invalidate its
+     * previous position.
+     */
+    if (term->dispcursy >= 0 &&
+       (term->curstype != cursor ||
+        term->dispcursy != our_curs_y ||
+        term->dispcursx != our_curs_x)) {
+       termchar *dispcurs = term->disptext[term->dispcursy]->chars +
+           term->dispcursx;
+
+       if (term->dispcursx > 0 && dispcurs->chr == UCSWIDE)
+           dispcurs[-1].attr |= ATTR_INVALID;
+       if (term->dispcursx < term->cols-1 && dispcurs[1].chr == UCSWIDE)
+           dispcurs[1].attr |= ATTR_INVALID;
+       dispcurs->attr |= ATTR_INVALID;
+
        term->curstype = 0;
     }
-    term->dispcurs = NULL;
+    term->dispcursx = term->dispcursy = -1;
+
+#ifdef OPTIMISE_SCROLL
+    /* Do scrolls */
+    sr = term->scrollhead;
+    while (sr) {
+       struct scrollregion *next = sr->next;
+       do_scroll(ctx, sr->topline, sr->botline, sr->lines);
+       sfree(sr);
+       sr = next;
+    }
+    term->scrollhead = term->scrolltail = NULL;
+#endif /* OPTIMISE_SCROLL */
 
     /* The normal screen data */
     for (i = 0; i < term->rows; i++) {
-       unsigned long *ldata;
-       int lattr;
-       int idx, dirty_line, dirty_run, selected;
-       unsigned long attr = 0;
+       termline *ldata;
+       termchar *lchars;
+       int dirty_line, dirty_run, selected;
+       unsigned long attr = 0, cset = 0;
        int updated_line = 0;
        int start = 0;
        int ccount = 0;
@@ -3160,36 +3992,96 @@ static void do_paint(Terminal *term, Context ctx, int may_optimise)
 
        scrpos.y = i + term->disptop;
        ldata = lineptr(scrpos.y);
-       lattr = (ldata[term->cols] & LATTR_MODE);
 
-       idx = i * (term->cols + 1);
-       dirty_run = dirty_line = (ldata[term->cols] !=
-                                 term->disptext[idx + term->cols]);
-       term->disptext[idx + term->cols] = ldata[term->cols];
+       dirty_run = dirty_line = (ldata->lattr !=
+                                 term->disptext[i]->lattr);
+       term->disptext[i]->lattr = ldata->lattr;
+
+       /* Do Arabic shaping and bidi. */
+       if(!term->cfg.bidi || !term->cfg.arabicshaping) {
 
-       for (j = 0; j < term->cols; j++, idx++) {
+           if (!term_bidi_cache_hit(term, i, ldata->chars, term->cols)) {
+
+               for(it=0; it<term->cols ; it++)
+               {
+                   unsigned long uc = (ldata->chars[it].chr);
+
+                   switch (uc & CSET_MASK) {
+                     case CSET_LINEDRW:
+                       if (!term->cfg.rawcnp) {
+                           uc = term->ucsdata->unitab_xterm[uc & 0xFF];
+                           break;
+                       }
+                     case CSET_ASCII:
+                       uc = term->ucsdata->unitab_line[uc & 0xFF];
+                       break;
+                     case CSET_SCOACS:
+                       uc = term->ucsdata->unitab_scoacs[uc&0xFF];
+                       break;
+                   }
+                   switch (uc & CSET_MASK) {
+                     case CSET_ACP:
+                       uc = term->ucsdata->unitab_font[uc & 0xFF];
+                       break;
+                     case CSET_OEMCP:
+                       uc = term->ucsdata->unitab_oemcp[uc & 0xFF];
+                       break;
+                   }
+
+                   term->wcFrom[it].origwc = term->wcFrom[it].wc =
+                       (wchar_t)uc;
+                   term->wcFrom[it].index = it;
+               }
+
+               if(!term->cfg.bidi)
+                   do_bidi(term->wcFrom, term->cols);
+
+               /* this is saved iff done from inside the shaping */
+               if(!term->cfg.bidi && term->cfg.arabicshaping)
+                   for(it=0; it<term->cols; it++)
+                       term->wcTo[it] = term->wcFrom[it];
+
+               if(!term->cfg.arabicshaping)
+                   do_shape(term->wcFrom, term->wcTo, term->cols);
+
+               for(it=0; it<term->cols ; it++)
+               {
+                   term->ltemp[it] = ldata->chars[term->wcTo[it].index];
+
+                   if (term->wcTo[it].origwc != term->wcTo[it].wc)
+                       term->ltemp[it].chr = term->wcTo[it].wc;
+               }
+               term_bidi_cache_store(term, i, ldata->chars,
+                                     term->ltemp, term->cols);
+               lchars = term->ltemp;
+           } else {
+               lchars = term->post_bidi_cache[i];
+           }
+       } else
+           lchars = ldata->chars;
+
+       for (j = 0; j < term->cols; j++) {
            unsigned long tattr, tchar;
-           unsigned long *d = ldata + j;
+           termchar *d = lchars + j;
            int break_run;
            scrpos.x = j;
 
-           tchar = (*d & (CHAR_MASK | CSET_MASK));
-           tattr = (*d & (ATTR_MASK ^ CSET_MASK));
+           tchar = d->chr;
+           tattr = d->attr;
+
            switch (tchar & CSET_MASK) {
-             case ATTR_ASCII:
+             case CSET_ASCII:
                tchar = term->ucsdata->unitab_line[tchar & 0xFF];
                break;
-             case ATTR_LINEDRW:
+             case CSET_LINEDRW:
                tchar = term->ucsdata->unitab_xterm[tchar & 0xFF];
                break;
-             case ATTR_SCOACS:  
+             case CSET_SCOACS:  
                tchar = term->ucsdata->unitab_scoacs[tchar&0xFF]; 
                break;
            }
-           tattr |= (tchar & CSET_MASK);
-           tchar &= CHAR_MASK;
-           if ((d[1] & (CHAR_MASK | CSET_MASK)) == UCSWIDE)
-                   tattr |= ATTR_WIDE;
+           if (j < term->cols-1 && d[1].chr == UCSWIDE)
+               tattr |= ATTR_WIDE;
 
            /* Video reversing things */
            if (term->selstate == DRAGGING || term->selstate == SELECTED) {
@@ -3216,31 +4108,42 @@ static void do_paint(Terminal *term, Context ctx, int may_optimise)
             * Check the font we'll _probably_ be using to see if 
             * the character is wide when we don't want it to be.
             */
-           if ((tchar | tattr) != (term->disptext[idx]& ~ATTR_NARROW)) {
-               if ((tattr & ATTR_WIDE) == 0 && 
-                   char_width(ctx, (tchar | tattr) & 0xFFFF) == 2)
+           if (tchar != term->disptext[i]->chars[j].chr ||
+               tattr != (term->disptext[i]->chars[j].attr &~
+                         ATTR_NARROW)) {
+               if ((tattr & ATTR_WIDE) == 0 && char_width(ctx, tchar) == 2)
                    tattr |= ATTR_NARROW;
-           } else if (term->disptext[idx]&ATTR_NARROW)
+           } else if (term->disptext[i]->chars[j].attr & ATTR_NARROW)
                tattr |= ATTR_NARROW;
 
            /* Cursor here ? Save the 'background' */
            if (i == our_curs_y && j == our_curs_x) {
-               cursor_background = tattr | tchar;
-               term->dispcurs = term->disptext + idx;
+               cursor_background.chr = tchar;
+               cursor_background.attr = tattr;
+               term->dispcursx = j;
+               term->dispcursy = i;
            }
 
-           if ((term->disptext[idx] ^ tattr) & ATTR_WIDE)
+           if ((term->disptext[i]->chars[j].attr ^ tattr) & ATTR_WIDE)
                dirty_line = TRUE;
 
            break_run = (((tattr ^ attr) & term->attr_mask) ||
                j - start >= sizeof(ch));
 
            /* Special hack for VT100 Linedraw glyphs */
-           if ((attr & CSET_MASK) == 0x2300 && tchar >= 0xBA
-               && tchar <= 0xBD) break_run = TRUE;
+           if (tchar >= 0x23BA && tchar <= 0x23BD)
+               break_run = TRUE;
+
+           /*
+            * Separate out sequences of characters that have the
+            * same CSET, if that CSET is a magic one.
+            */
+           if (CSET_OF(tchar) != cset)
+               break_run = TRUE;
 
            if (!term->ucsdata->dbcs_screenfont && !dirty_line) {
-               if ((tchar | tattr) == term->disptext[idx])
+               if (term->disptext[i]->chars[j].chr == tchar &&
+                   term->disptext[i]->chars[j].attr == tattr)
                    break_run = TRUE;
                else if (!dirty_run && ccount == 1)
                    break_run = TRUE;
@@ -3248,26 +4151,28 @@ static void do_paint(Terminal *term, Context ctx, int may_optimise)
 
            if (break_run) {
                if ((dirty_run || last_run_dirty) && ccount > 0) {
-                   do_text(ctx, start, i, ch, ccount, attr, lattr);
+                   do_text(ctx, start, i, ch, ccount, attr, ldata->lattr);
                    updated_line = 1;
                }
                start = j;
                ccount = 0;
                attr = tattr;
+               cset = CSET_OF(tchar);
                if (term->ucsdata->dbcs_screenfont)
                    last_run_dirty = dirty_run;
                dirty_run = dirty_line;
            }
 
-           if ((tchar | tattr) != term->disptext[idx])
+           if (term->disptext[i]->chars[j].chr != tchar ||
+               term->disptext[i]->chars[j].attr != tattr)
                dirty_run = TRUE;
-           ch[ccount++] = (char) tchar;
-           term->disptext[idx] = tchar | tattr;
+           ch[ccount++] = (wchar_t) tchar;
+           term->disptext[i]->chars[j].chr = tchar;
+           term->disptext[i]->chars[j].attr = tattr;
 
            /* If it's a wide char step along to the next one. */
            if (tattr & ATTR_WIDE) {
                if (++j < term->cols) {
-                   idx++;
                    d++;
                    /*
                     * By construction above, the cursor should not
@@ -3275,24 +4180,27 @@ static void do_paint(Terminal *term, Context ctx, int may_optimise)
                     * Ever.
                     */
                    assert(!(i == our_curs_y && j == our_curs_x));
-                   if (term->disptext[idx] != *d)
+                   if (memcmp(&term->disptext[i]->chars[j],
+                              d, sizeof(*d)))
                        dirty_run = TRUE;
-                   term->disptext[idx] = *d;
+                   term->disptext[i]->chars[j] = *d;
                }
            }
        }
        if (dirty_run && ccount > 0) {
-           do_text(ctx, start, i, ch, ccount, attr, lattr);
+           do_text(ctx, start, i, ch, ccount, attr, ldata->lattr);
            updated_line = 1;
        }
 
        /* Cursor on this line ? (and changed) */
        if (i == our_curs_y && (term->curstype != cursor || updated_line)) {
-           ch[0] = (char) (cursor_background & CHAR_MASK);
-           attr = (cursor_background & ATTR_MASK) | cursor;
-           do_cursor(ctx, our_curs_x, i, ch, 1, attr, lattr);
+           ch[0] = (wchar_t) cursor_background.chr;
+           attr = cursor_background.attr | cursor;
+           do_cursor(ctx, our_curs_x, i, ch, 1, attr, ldata->lattr);
            term->curstype = cursor;
        }
+
+       unlineptr(ldata);
     }
 }
 
@@ -3334,10 +4242,11 @@ void term_blink(Terminal *term, int flg)
  */
 void term_invalidate(Terminal *term)
 {
-    int i;
+    int i, j;
 
-    for (i = 0; i < term->rows * (term->cols + 1); i++)
-       term->disptext[i] = ATTR_INVALID;
+    for (i = 0; i < term->rows; i++)
+       for (j = 0; j < term->cols; j++)
+           term->disptext[i]->chars[j].attr = ATTR_INVALID;
 }
 
 /*
@@ -3353,13 +4262,12 @@ void term_paint(Terminal *term, Context ctx,
     if (bottom >= term->rows) bottom = term->rows-1;
 
     for (i = top; i <= bottom && i < term->rows; i++) {
-       if ((term->disptext[i * (term->cols + 1) + term->cols] &
-            LATTR_MODE) == LATTR_NORM)
+       if (term->disptext[i]->lattr == LATTR_NORM)
            for (j = left; j <= right && j < term->cols; j++)
-               term->disptext[i * (term->cols + 1) + j] = ATTR_INVALID;
+               term->disptext[i]->chars[j].attr = ATTR_INVALID;
        else
            for (j = left / 2; j <= right / 2 + 1 && j < term->cols; j++)
-               term->disptext[i * (term->cols + 1) + j] = ATTR_INVALID;
+               term->disptext[i]->chars[j].attr = ATTR_INVALID;
     }
 
     /* This should happen soon enough, also for some reason it sometimes 
@@ -3379,7 +4287,7 @@ void term_paint(Terminal *term, Context ctx,
  */
 void term_scroll(Terminal *term, int rel, int where)
 {
-    int sbtop = -count234(term->scrollback);
+    int sbtop = -sblines(term);
 #ifdef OPTIMISE_SCROLL
     int olddisptop = term->disptop;
     int shift;
@@ -3399,7 +4307,7 @@ void term_scroll(Terminal *term, int rel, int where)
     term_update(term);
 }
 
-static void clipme(Terminal *term, pos top, pos bottom, int rect)
+static void clipme(Terminal *term, pos top, pos bottom, int rect, int desel)
 {
     wchar_t *workbuf;
     wchar_t *wbptr;                   /* where next char goes within workbuf */
@@ -3408,13 +4316,13 @@ static void clipme(Terminal *term, pos top, pos bottom, int rect)
     int buflen;                               /* amount of memory allocated to workbuf */
 
     buflen = 5120;                    /* Default size */
-    workbuf = smalloc(buflen * sizeof(wchar_t));
+    workbuf = snewn(buflen, wchar_t);
     wbptr = workbuf;                  /* start filling here */
     old_top_x = top.x;                /* needed for rect==1 */
 
     while (poslt(top, bottom)) {
        int nl = FALSE;
-       unsigned long *ldata = lineptr(top.y);
+       termline *ldata = lineptr(top.y);
        pos nlpos;
 
        /*
@@ -3431,15 +4339,13 @@ static void clipme(Terminal *term, pos top, pos bottom, int rect)
         * because in normal selection mode this means we need a
         * newline at the end)...
         */
-       if (!(ldata[term->cols] & LATTR_WRAPPED)) {
-           while (((ldata[nlpos.x - 1] & 0xFF) == 0x20 ||
-                   (DIRECT_CHAR(ldata[nlpos.x - 1]) &&
-                    (ldata[nlpos.x - 1] & CHAR_MASK) == 0x20))
-                  && poslt(top, nlpos))
+       if (!(ldata->lattr & LATTR_WRAPPED)) {
+           while (IS_SPACE_CHR(ldata->chars[nlpos.x - 1].chr) &&
+                  poslt(top, nlpos))
                decpos(nlpos);
            if (poslt(nlpos, bottom))
                nl = TRUE;
-       } else if (ldata[term->cols] & LATTR_WRAPPED2) {
+       } else if (ldata->lattr & LATTR_WRAPPED2) {
            /* Ignore the last char on the line in a WRAPPED2 line. */
            decpos(nlpos);
        }
@@ -3463,7 +4369,7 @@ static void clipme(Terminal *term, pos top, pos bottom, int rect)
            sprintf(cbuf, "<U+%04x>", (ldata[top.x] & 0xFFFF));
 #else
            wchar_t cbuf[16], *p;
-           int uc = (ldata[top.x] & 0xFFFF);
+           int uc = ldata->chars[top.x].chr;
            int set, c;
 
            if (uc == UCSWIDE) {
@@ -3472,29 +4378,29 @@ static void clipme(Terminal *term, pos top, pos bottom, int rect)
            }
 
            switch (uc & CSET_MASK) {
-             case ATTR_LINEDRW:
+             case CSET_LINEDRW:
                if (!term->cfg.rawcnp) {
                    uc = term->ucsdata->unitab_xterm[uc & 0xFF];
                    break;
                }
-             case ATTR_ASCII:
+             case CSET_ASCII:
                uc = term->ucsdata->unitab_line[uc & 0xFF];
                break;
-             case ATTR_SCOACS:  
+             case CSET_SCOACS:  
                uc = term->ucsdata->unitab_scoacs[uc&0xFF]; 
                break;
            }
            switch (uc & CSET_MASK) {
-             case ATTR_ACP:
+             case CSET_ACP:
                uc = term->ucsdata->unitab_font[uc & 0xFF];
                break;
-             case ATTR_OEMCP:
+             case CSET_OEMCP:
                uc = term->ucsdata->unitab_oemcp[uc & 0xFF];
                break;
            }
 
            set = (uc & CSET_MASK);
-           c = (uc & CHAR_MASK);
+           c = (uc & ~CSET_MASK);
            cbuf[0] = uc;
            cbuf[1] = 0;
 
@@ -3505,7 +4411,7 @@ static void clipme(Terminal *term, pos top, pos bottom, int rect)
                    int rv;
                    if (is_dbcs_leadbyte(term->ucsdata->font_codepage, (BYTE) c)) {
                        buf[0] = c;
-                       buf[1] = (char) (0xFF & ldata[top.x + 1]);
+                       buf[1] = (char) (0xFF & ldata->chars[top.x + 1].chr);
                        rv = mb_to_wc(term->ucsdata->font_codepage, 0, buf, 2, wbuf, 4);
                        top.x++;
                    } else {
@@ -3524,9 +4430,8 @@ static void clipme(Terminal *term, pos top, pos bottom, int rect)
            for (p = cbuf; *p; p++) {
                /* Enough overhead for trailing NL and nul */
                if (wblen >= buflen - 16) {
-                   workbuf =
-                       srealloc(workbuf,
-                                sizeof(wchar_t) * (buflen += 100));
+                   buflen += 100;
+                   workbuf = sresize(workbuf, buflen, wchar_t);
                    wbptr = workbuf + wblen;
                }
                wblen++;
@@ -3543,12 +4448,14 @@ static void clipme(Terminal *term, pos top, pos bottom, int rect)
        }
        top.y++;
        top.x = rect ? old_top_x : 0;
+
+       unlineptr(ldata);
     }
 #if SELECTION_NUL_TERMINATED
     wblen++;
     *wbptr++ = 0;
 #endif
-    write_clip(term->frontend, workbuf, wblen, FALSE); /* transfer to clipbd */
+    write_clip(term->frontend, workbuf, wblen, desel); /* transfer to clipbd */
     if (buflen > 0)                   /* indicates we allocated this buffer */
        sfree(workbuf);
 }
@@ -3556,9 +4463,13 @@ static void clipme(Terminal *term, pos top, pos bottom, int rect)
 void term_copyall(Terminal *term)
 {
     pos top;
-    top.y = -count234(term->scrollback);
+    pos bottom;
+    tree234 *screen = term->screen;
+    top.y = -sblines(term);
     top.x = 0;
-    clipme(term, top, term->curs, 0);
+    bottom.y = find_last_nonempty_line(term, screen);
+    bottom.x = term->cols;
+    clipme(term, top, bottom, 0, TRUE);
 }
 
 /*
@@ -3636,29 +4547,27 @@ static int wordtype(Terminal *term, int uc)
     };
     const struct ucsword *wptr;
 
-    uc &= (CSET_MASK | CHAR_MASK);
-
     switch (uc & CSET_MASK) {
-      case ATTR_LINEDRW:
+      case CSET_LINEDRW:
        uc = term->ucsdata->unitab_xterm[uc & 0xFF];
        break;
-      case ATTR_ASCII:
+      case CSET_ASCII:
        uc = term->ucsdata->unitab_line[uc & 0xFF];
        break;
-      case ATTR_SCOACS:  
+      case CSET_SCOACS:  
        uc = term->ucsdata->unitab_scoacs[uc&0xFF]; 
        break;
     }
     switch (uc & CSET_MASK) {
-      case ATTR_ACP:
+      case CSET_ACP:
        uc = term->ucsdata->unitab_font[uc & 0xFF];
        break;
-      case ATTR_OEMCP:
+      case CSET_OEMCP:
        uc = term->ucsdata->unitab_oemcp[uc & 0xFF];
        break;
     }
 
-    /* For DBCS font's I can't do anything usefull. Even this will sometimes
+    /* For DBCS fonts I can't do anything useful. Even this will sometimes
      * fail as there's such a thing as a double width space. :-(
      */
     if (term->ucsdata->dbcs_screenfont &&
@@ -3681,9 +4590,9 @@ static int wordtype(Terminal *term, int uc)
  */
 static pos sel_spread_half(Terminal *term, pos p, int dir)
 {
-    unsigned long *ldata;
+    termline *ldata;
     short wvalue;
-    int topy = -count234(term->scrollback);
+    int topy = -sblines(term);
 
     ldata = lineptr(p.y);
 
@@ -3693,14 +4602,14 @@ static pos sel_spread_half(Terminal *term, pos p, int dir)
         * In this mode, every character is a separate unit, except
         * for runs of spaces at the end of a non-wrapping line.
         */
-       if (!(ldata[term->cols] & LATTR_WRAPPED)) {
-           unsigned long *q = ldata + term->cols;
-           while (q > ldata && (q[-1] & CHAR_MASK) == 0x20)
+       if (!(ldata->lattr & LATTR_WRAPPED)) {
+           termchar *q = ldata->chars + term->cols;
+           while (q > ldata->chars && IS_SPACE_CHR(q[-1].chr))
                q--;
-           if (q == ldata + term->cols)
+           if (q == ldata->chars + term->cols)
                q--;
-           if (p.x >= q - ldata)
-               p.x = (dir == -1 ? q - ldata : term->cols - 1);
+           if (p.x >= q - ldata->chars)
+               p.x = (dir == -1 ? q - ldata->chars : term->cols - 1);
        }
        break;
       case SM_WORD:
@@ -3708,26 +4617,30 @@ static pos sel_spread_half(Terminal *term, pos p, int dir)
         * In this mode, the units are maximal runs of characters
         * whose `wordness' has the same value.
         */
-       wvalue = wordtype(term, UCSGET(ldata, p.x));
+       wvalue = wordtype(term, UCSGET(ldata->chars, p.x));
        if (dir == +1) {
            while (1) {
-               int maxcols = (ldata[term->cols] & LATTR_WRAPPED2 ?
+               int maxcols = (ldata->lattr & LATTR_WRAPPED2 ?
                               term->cols-1 : term->cols);
                if (p.x < maxcols-1) {
-                   if (wordtype(term, UCSGET(ldata, p.x + 1)) == wvalue)
+                   if (wordtype(term, UCSGET(ldata->chars, p.x+1)) == wvalue)
                        p.x++;
                    else
                        break;
                } else {
-                   if (ldata[term->cols] & LATTR_WRAPPED) {
-                       unsigned long *ldata2;
+                   if (ldata->lattr & LATTR_WRAPPED) {
+                       termline *ldata2;
                        ldata2 = lineptr(p.y+1);
-                       if (wordtype(term, UCSGET(ldata2, 0)) == wvalue) {
+                       if (wordtype(term, UCSGET(ldata2->chars, 0))
+                           == wvalue) {
                            p.x = 0;
                            p.y++;
+                           unlineptr(ldata);
                            ldata = ldata2;
-                       } else
+                       } else {
+                           unlineptr(ldata2);
                            break;
+                       }
                    } else
                        break;
                }
@@ -3735,26 +4648,29 @@ static pos sel_spread_half(Terminal *term, pos p, int dir)
        } else {
            while (1) {
                if (p.x > 0) {
-                   if (wordtype(term, UCSGET(ldata, p.x - 1)) == wvalue)
+                   if (wordtype(term, UCSGET(ldata->chars, p.x-1)) == wvalue)
                        p.x--;
                    else
                        break;
                } else {
-                   unsigned long *ldata2;
+                   termline *ldata2;
                    int maxcols;
                    if (p.y <= topy)
                        break;
                    ldata2 = lineptr(p.y-1);
-                   maxcols = (ldata2[term->cols] & LATTR_WRAPPED2 ?
+                   maxcols = (ldata2->lattr & LATTR_WRAPPED2 ?
                              term->cols-1 : term->cols);
-                   if (ldata2[term->cols] & LATTR_WRAPPED) {
-                       if (wordtype(term, UCSGET(ldata2, maxcols-1))
+                   if (ldata2->lattr & LATTR_WRAPPED) {
+                       if (wordtype(term, UCSGET(ldata2->chars, maxcols-1))
                            == wvalue) {
                            p.x = maxcols-1;
                            p.y--;
+                           unlineptr(ldata);
                            ldata = ldata2;
-                       } else
+                       } else {
+                           unlineptr(ldata2);
                            break;
+                       }
                    } else
                        break;
                }
@@ -3768,6 +4684,8 @@ static pos sel_spread_half(Terminal *term, pos p, int dir)
        p.x = (dir == -1 ? 0 : term->cols - 1);
        break;
     }
+
+    unlineptr(ldata);
     return p;
 }
 
@@ -3795,7 +4713,7 @@ void term_do_paste(Terminal *term)
         if (term->paste_buffer)
             sfree(term->paste_buffer);
         term->paste_pos = term->paste_hold = term->paste_len = 0;
-        term->paste_buffer = smalloc(len * sizeof(wchar_t));
+        term->paste_buffer = snewn(len, wchar_t);
 
         p = q = data;
         while (p < data + len) {
@@ -3836,7 +4754,7 @@ void term_mouse(Terminal *term, Mouse_Button braw, Mouse_Button bcooked,
                Mouse_Action a, int x, int y, int shift, int ctrl, int alt)
 {
     pos selpoint;
-    unsigned long *ldata;
+    termline *ldata;
     int raw_mouse = (term->xterm_mouse &&
                     !term->cfg.no_mouse_rep &&
                     !(term->cfg.mouse_override && shift));
@@ -3865,8 +4783,9 @@ void term_mouse(Terminal *term, Mouse_Button braw, Mouse_Button bcooked,
     selpoint.y = y + term->disptop;
     selpoint.x = x;
     ldata = lineptr(selpoint.y);
-    if ((ldata[term->cols] & LATTR_MODE) != LATTR_NORM)
+    if ((ldata->lattr & LATTR_MODE) != LATTR_NORM)
        selpoint.x /= 2;
+    unlineptr(ldata);
 
     if (raw_mouse) {
        int encstate = 0, r, c;
@@ -4023,7 +4942,7 @@ void term_mouse(Terminal *term, Mouse_Button braw, Mouse_Button bcooked,
             * data to the clipboard.
             */
            clipme(term, term->selstart, term->selend,
-                  (term->seltype == RECTANGULAR));
+                  (term->seltype == RECTANGULAR), FALSE);
            term->selstate = SELECTED;
        } else
            term->selstate = NO_SELECTION;
@@ -4050,7 +4969,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
 
     fprintf(stderr, "keysym = %d, %d chars:", keysym, tlen);
     for (i = 0; i < tlen; i++)
-       fprintf(stderr, " %04x", text[i]);
+       fprintf(stderr, " %04x", (unsigned)text[i]);
     fprintf(stderr, "\n");
 #endif
 
@@ -4099,8 +5018,8 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
     if (keysym == PK_NULL && (modifiers & PKM_CONTROL) && tlen == 1 &&
        text[0] >= 0x20 && text[0] <= 0x7e) {
        /* ASCII chars + Control */
-       if (text[0] >= 0x40 && text[0] <= 0x5f ||
-           text[0] >= 0x61 && text[0] <= 0x7a)
+       if ((text[0] >= 0x40 && text[0] <= 0x5f) ||
+           (text[0] >= 0x61 && text[0] <= 0x7a))
            text[0] &= 0x1f;
        else {
            /*
@@ -4139,6 +5058,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
          case PK_KP7: c = 'y'; break;
          case PK_KP8: c = 'k'; break;
          case PK_KP9: c = 'u'; break;
+         default: break; /* else gcc warns `enum value not used' */
        }
        if (c != 0) {
            if (c != '.') {
@@ -4169,6 +5089,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
              case PK_PF2: xkey = 'Q'; break;
              case PK_PF3: xkey = 'R'; break;
              case PK_PF4: xkey = 'S'; break;
+             default: break; /* else gcc warns `enum value not used' */
            }
        }
        if (term->app_keypad_keys && !term->cfg.no_applic_k) {
@@ -4185,6 +5106,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
              case PK_KP9: xkey = 'y'; break;
              case PK_KPDECIMAL: xkey = 'n'; break;
              case PK_KPENTER: xkey = 'M'; break;
+             default: break; /* else gcc warns `enum value not used' */
            }
            if (term->cfg.funky_type == FUNKY_XTERM && tlen > 0) {
                /*
@@ -4219,6 +5141,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
                    break;
                  case PK_KPMINUS: xkey = 'm'; break;
                  case PK_KPCOMMA: xkey = 'l'; break;
+                 default: break; /* else gcc warns `enum value not used' */
                }
            }
        }
@@ -4245,6 +5168,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
              case PK_KP7: keysym = PK_HOME; break;
              case PK_KP8: keysym = PK_UP; break;
              case PK_KP9: keysym = PK_PAGEUP; break;
+             default: break; /* else gcc warns `enum value not used' */
            }
        }
     }
@@ -4283,6 +5207,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
                *p++ = 0x0a;
            goto done;
        }
+      default: break; /* else gcc warns `enum value not used' */
     }
 
     /* SCO function keys and editing keys */
@@ -4307,6 +5232,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
              case PK_END:      xkey = 'F'; break;
              case PK_PAGEUP:   xkey = 'I'; break;
              case PK_PAGEDOWN: xkey = 'G'; break;
+             default: break; /* else gcc warns `enum value not used' */
            }
            p += sprintf((char *) p, "\x1B[%c", xkey);
        }
@@ -4324,6 +5250,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
              case PK_END:      keysym = PK_PAGEUP;   break;
              case PK_PAGEUP:   keysym = PK_DELETE;   break;
              case PK_PAGEDOWN: keysym = PK_PAGEDOWN; break;
+             default: break; /* else gcc warns `enum value not used' */
            }
        }
 
@@ -4348,6 +5275,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
              case PK_END:      xkey = 'E'; break;
              case PK_PAGEUP:   xkey = 'I'; break;
              case PK_PAGEDOWN: xkey = 'G'; break;
+             default: xkey=0; break; /* else gcc warns `enum value not used'*/
            }
            p += sprintf((char *) p, "\x1B%c", xkey);
            goto done;
@@ -4360,6 +5288,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
          case PK_END:      code = 4; break;
          case PK_PAGEUP:   code = 5; break;
          case PK_PAGEDOWN: code = 6; break;
+         default: code = 0; break; /* else gcc warns `enum value not used' */
        }
        p += sprintf((char *) p, "\x1B[%d~", code);
        goto done;
@@ -4403,6 +5332,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
          case PK_RIGHT: xkey = 'C'; break;
          case PK_LEFT:  xkey = 'D'; break;
          case PK_REST:  xkey = 'G'; break; /* centre key on number pad */
+         default: xkey = 0; break; /* else gcc warns `enum value not used' */
        }
        if (term->vt52_mode)
            p += sprintf((char *) p, "\x1B%c", xkey);
@@ -4460,7 +5390,7 @@ void term_key(Terminal *term, Key_Sym keysym, wchar_t *text, size_t tlen,
 #if 0
            fprintf(stderr, "sending %d unichars:", tlen);
            for (i = 0; i < tlen; i++)
-               fprintf(stderr, " %04x", text[i]);
+               fprintf(stderr, " %04x", (unsigned) text[i]);
            fprintf(stderr, "\n");
 #endif
            luni_send(term->ldisc, text, tlen, 1);
@@ -4539,17 +5469,17 @@ int term_ldisc(Terminal *term, int option)
     return FALSE;
 }
 
-/*
- * from_backend(), to get data from the backend for the terminal.
- */
-int from_backend(void *vterm, int is_stderr, char *data, int len)
+int term_data(Terminal *term, int is_stderr, const char *data, int len)
 {
-    Terminal *term = (Terminal *)vterm;
-
-    assert(len > 0);
-
     bufchain_add(&term->inbuf, data, len);
 
+    if (!term->in_term_out) {
+       term->in_term_out = TRUE;
+       term_blink(term, 1);
+       term_out(term);
+       term->in_term_out = FALSE;
+    }
+
     /*
      * term_out() always completely empties inbuf. Therefore,
      * there's no reason at all to return anything other than zero