sled

чување 91cb8cdd809851a80e37200744c6f0fb0d2e51e3
родитељ 69b585a99f4b85695f5424be1dbe81937a3765b5
Аутор: Страхиња Радић <sr@strahinja.org>
Датум:   Fri,  9 Aug 2024 12:02:52 +0200

patch/syntax.patch: Use a different strategy with regards to 32-bit 
Unicode, wchar_t and char's in keyword/operator/comment delimiter lists

Diffstat:
M
patch/syntax.patch
 | 
375
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----------------

измењених датотека: 1, додавања: 294(+), брисања: 81(-)
diff --git a/patch/syntax.patch b/patch/syntax.patch
@@ -1,6 +1,6 @@
-From 9fba930eb350fc45fab6d21c12fb9efc1d9c18fe Mon Sep 17 00:00:00 2001
+From 167fce2e349dbac434cbbeeda3e1744280e12d83 Mon Sep 17 00:00:00 2001
 From: Страхиња Радић <sr@strahinja.org>
-Date: Thu Aug  8 19:35:11 2024 UTC
+Date: Fri Aug  9 09:59:59 2024 UTC
 Subject: [PATCH] Add syntax highlighting
 MIME-Version: 1.0
 Content-Type: text/plain; charset=UTF-8
@@ -8,13 +8,13 @@ Content-Transfer-Encoding: 8bit
 
 ---
  config.def.h  |    7+   0-
- draw.c        |  487+  29-
- draw.h        |   63+   6-
+ draw.c        |  668+  34-
+ draw.h        |   82+   6-
  sled.c        |   82+  16-
- 4 files changed, 639 insertions(+), 51 deletions(-)
+ 4 files changed, 839 insertions(+), 56 deletions(-)
 
 diff --git a/config.def.h b/config.def.h
-index f72c9de89..9fba930eb 100644
+index 69b585a99..167fce2e3 100644
 --- a/config.def.h
 +++ b/config.def.h
 @@ -239,6 +239,13 @@ static const struct Key command_keys[] = {
@@ -32,7 +32,7 @@ index f72c9de89..9fba930eb 100644
  	{.format    = "%s%s",
  	 .callback  = format_filename,
 diff --git a/draw.c b/draw.c
-index f72c9de89..9fba930eb 100644
+index 69b585a99..167fce2e3 100644
 --- a/draw.c
 +++ b/draw.c
 @@ -3,6 +3,7 @@
@@ -56,7 +56,7 @@ index f72c9de89..9fba930eb 100644
 +#define SLC(com)                                                             \
 +	{                                                                    \
 +		.start_text = com, .start_size = sizeof(com) / sizeof(*com), \
-+		.end_text = NULL, .end_size = -1                             \
++		.end_text = NULL, .end_size = 0                              \
 +	}
 +/* Area comment specification macro */
 +#define AC(com1, com2)                                                        \
@@ -84,12 +84,12 @@ index f72c9de89..9fba930eb 100644
 +	KW("typedef"), KW("union"), KW("unsigned"), KW("void"),
 +	KW("volatile"), KW("while"), KW("_Bool"), KW("_Complex"),
 +	KW("_Imaginary"),
-+	{ .text = NULL }
++	{ .text = NULL,		.size = 0 }
 +};
 +const struct Comment c_comments[] = {
 +	SLC("//"),
 +	AC("/*", "*/"),
-+	{ .start_text = NULL }
++	{ .start_text = NULL,	.start_size = 0 }
 +};
 +const struct Operator c_operators[] = {
 +	KW("<<="), KW(">>="), KW("<<"), KW(">>"),
@@ -104,7 +104,7 @@ index f72c9de89..9fba930eb 100644
 +	KW("<"), KW(">"),
 +	KW(","), KW(";"), KW(":"), KW("?"), KW("."),
 +	KW("("), KW(")"), KW("["), KW("]"), KW("{"), KW("}"),
-+	{ .text = NULL }
++	{ .text = NULL,		.size = 0 }
 +};
 +const struct Keyword shell_keywords[]	= {
 +	KW("!"), KW("{"), KW("}"),
@@ -149,18 +149,18 @@ index f72c9de89..9fba930eb 100644
 +	KW("break"), KW(":"), KW("."), KW("continue"), KW("eval"),
 +	KW("exec"), KW("exit"), KW("export"), KW("readonly"), KW("return"),
 +	KW("set"), KW("shift"), KW("times"), KW("trap"), KW("unset"),
-+	{.text = NULL}
++	{ .text = NULL,		.size = 0 }
 +};
 +const struct Comment shell_comments[]	= {
 +	SLC("#"),
-+	{.start_text = NULL}
++	{ .start_text = NULL,	.start_size = 0 }
 +};
 +const struct Operator shell_operators[] = {
 +	KW("<<"), KW(">>"),
 +	KW("&&"), KW("||"), KW("$"), KW("`"), KW("="),
 +	KW("<"), KW(">"),
 +	KW(";"), KW("("), KW(")"), KW("["), KW("]"), KW("{"), KW("}"),
-+	{.text = NULL}
++	{ .text = NULL,		.size = 0 }
 +};
  const char* go_to_dialog_title   = "Go to line";
  const char* prompt_dirty	 = "File changed, save [y/n/ESC]?";
@@ -270,7 +270,7 @@ index f72c9de89..9fba930eb 100644
 -				0, 0, LEFT, fsc, 0, state->search, SEARCH_FG,
 -				SEARCH_BG, ssi, sei, SEL_FG, SEL_BG);
 +				cy, TEXT_FG, TEXT_BG, current->text,
-+				current->syntax, current->syntax_length, maxx,
++				current->syntax, current->syntax_size, maxx,
 +				maxx, 0, 0, LEFT, fsc, 0, state->search,
 +				SEARCH_FG, SEARCH_BG, ssi, sei, SEL_FG, SEL_BG);
  		i++;
@@ -337,7 +337,7 @@ index f72c9de89..9fba930eb 100644
  	free(us);
  	return 0;
  }
-@@ -639,6 +791,8 @@ free_bufferline(struct BufferLine* bl)
+@@ -639,14 +791,41 @@ free_bufferline(struct BufferLine* bl)
  	bl->length = 0;
  	bl->fg	   = 0;
  	bl->bg	   = 0;
@@ -346,15 +346,59 @@ index f72c9de89..9fba930eb 100644
  }
  
  void
-@@ -677,6 +831,7 @@ init_bufferline(struct BufferLine* bl)
- 	bl->size   = 0;
- 	bl->fg	   = 0;
- 	bl->bg	   = 0;
-+	bl->syntax = NULL;
+ free_drawstate(struct DrawState* state, const int skip_paste_free,
+ 	const int skip_filename_free)
+ {
++	struct Keyword32* kw;
++	struct Comment32* cm;
++	struct Operator32* op;
+ 	ssize_t row = 0;
+ 	assert(state != NULL);
++
++	if (state->ukeywords)
++	{
++		for (kw = state->ukeywords; kw->text != NULL; kw++)
++			free(kw->text);
++		free(state->ukeywords);
++	}
++	if (state->ucomments)
++	{
++		for (cm = state->ucomments; cm->start_text != NULL; cm++)
++		{
++			free(cm->start_text);
++			free(cm->end_text);
++		}
++		free(state->ucomments);
++	}
++	if (state->uoperators)
++	{
++		for (op = state->uoperators; op->text != NULL; op++)
++			free(op->text);
++		free(state->uoperators);
++	}
+ 	for (row = 0; row < state->buffer_size; row++)
+ 		free_bufferline(state->buffer + row);
+ 	free(state->buffer);
+@@ -672,11 +851,13 @@ init_bufferline(struct BufferLine* bl)
+ {
+ 	if (!bl)
+ 		return;
+-	bl->text   = NULL;
+-	bl->length = 0;
+-	bl->size   = 0;
+-	bl->fg	   = 0;
+-	bl->bg	   = 0;
++	bl->text	= NULL;
++	bl->length	= 0;
++	bl->size	= 0;
++	bl->fg		= 0;
++	bl->bg		= 0;
++	bl->syntax	= NULL;
++	bl->syntax_size = 0;
  }
  
  void
-@@ -702,6 +857,11 @@ init_drawstate(struct DrawState* state, char* error, c
+@@ -702,6 +883,14 @@ init_drawstate(struct DrawState* state, char* error, c
  		state->orig_real_filename = NULL;
  		state->backup_filename	  = NULL;
  		state->rfn_len		  = 0;
@@ -363,10 +407,13 @@ index f72c9de89..9fba930eb 100644
 +		state->keywords		  = NULL;
 +		state->comments		  = NULL;
 +		state->operators	  = NULL;
++		state->ukeywords	  = NULL;
++		state->ucomments	  = NULL;
++		state->uoperators	  = NULL;
  	}
  	state->input	    = NULL;
  	state->search	    = NULL;
-@@ -760,7 +920,282 @@ recalculate_size(struct DrawState* state)
+@@ -760,7 +949,429 @@ recalculate_size(struct DrawState* state)
  	recalculate_line_numbers(state);
  }
  
@@ -378,31 +425,46 @@ index f72c9de89..9fba930eb 100644
 +	struct SyntaxRegion* newreg;
 +
 +	assert(bl != NULL);
-+	if (bl->syntax_length + 1 >= bl->syntax_size)
-+	{
-+		temps = reallocarray(bl->syntax, bl->syntax_size + 1,
-+			sizeof(struct SyntaxRegion));
-+		if (!temps)
-+			return -1;
-+		bl->syntax = temps;
-+		bl->syntax_size++;
-+	}
-+	newreg		 = &bl->syntax[bl->syntax_length];
++	temps = reallocarray(bl->syntax, bl->syntax_size + 1,
++		sizeof(struct SyntaxRegion));
++	if (!temps)
++		return -1;
++	bl->syntax = temps;
++	bl->syntax_size++;
++	newreg		 = &bl->syntax[bl->syntax_size - 1];
 +	newreg->start	 = start;
 +	newreg->length	 = length;
 +	newreg->color.fg = syntax_colors[type].fg;
 +	newreg->color.bg = syntax_colors[type].bg;
-+	bl->syntax_length++;
 +	return 0;
 +}
 +
++/*
++ * Regenerate syntax regions from the document text.
++ * This is more complicated than it should be due to several factors:
++ *   - We are using 32-bit Unicode strings internally (uint32_t*)
++ *   - For non-ASCII, C99 supports only wchar_t string literals L"string" (if we
++ *     don't consider using UTF-8 everywhere, which would complicate everything
++ *     dealing with text and introduce numerous potential bugs)
++ *   - There are some systems out there (Windows) which use 16-bit wchar_t since
++ *     the standard is disappointingly vague, so we can't rely on seamless
++ *     conversion from wchar_t* to uint32_t*. Using code as-is leads to infinite
++ *     loop hangs.
++ * Thus, we can't use L"literals" to define the list of keywords, operators and
++ * comment delimiters, so we can either always convert them to uint32_t*, which
++ * is severely suboptimal performance-wise, or do what seems to be the most
++ * optimal solution, applied here: convert everything once and keep separate
++ * arrays with keywords, operators and comment delimiters explicitly converted
++ * to uint32_t*. This comes at the expense of more code bloat.
++ */
  void
 +recalculate_syntax(struct DrawState* state)
 +{
 +	struct BufferLine* curline;
-+	struct Keyword* keyword;
-+	struct Comment* comment;
-+	struct Operator* operator;
++	struct Keyword32* keyword;
++	struct Comment32* comment;
++	struct Operator32* operator;
++	uint32_t* temp32;
 +	char* temp;
 +	ssize_t i, li, ti, comment_start, quote_start, literal_start;
 +	uint32_t quote;
@@ -413,15 +475,162 @@ index f72c9de89..9fba930eb 100644
 +		&& (state->operators != NULL)
 +		&& (state->buffer[state->first_shown.row].text != NULL));
 +
++	if (!state->ukeywords)
++	{
++		struct Keyword* kw = state->keywords;
++		struct Keyword32* pklist;
++		size_t keyword_count = 0;
++		while (kw->text != NULL)
++		{
++			keyword_count++;
++			kw++;
++		}
++		assert(keyword_count > 0);
++		state->ukeywords
++			= calloc(keyword_count + 1, sizeof(struct Keyword32));
++		if (!state->ukeywords)
++		{
++			MEMCCPY(state->error, errors[ERR_CANT_ALLOC],
++				LINE_DEFAULT, temp);
++			state->running = 0;
++			return;
++		}
++		pklist = state->ukeywords;
++		kw     = state->keywords;
++		while (kw->text != NULL)
++		{
++			uint32_t ukw[kw->size];
++			u8_string_to_unicode(ukw, kw->text, kw->size - 1);
++			pklist->text = calloc(kw->size, sizeof(*ukw));
++			if (!pklist->text)
++			{
++				MEMCCPY(state->error, errors[ERR_CANT_ALLOC],
++					LINE_DEFAULT, temp);
++				state->running = 0;
++				return;
++			}
++			U32_MEMCCPY(pklist->text, ukw, kw->size, temp32);
++			pklist->size = kw->size;
++			kw++;
++			pklist++;
++		}
++		pklist->text = NULL;
++		pklist->size = -1;
++	}
++	if (!state->ucomments)
++	{
++		struct Comment* cm = state->comments;
++		struct Comment32* pclist;
++		size_t comment_count = 0;
++		while (cm->start_text != NULL)
++		{
++			comment_count++;
++			cm++;
++		}
++		assert(comment_count > 0);
++		state->ucomments
++			= calloc(comment_count + 1, sizeof(struct Comment32));
++		if (!state->ucomments)
++		{
++			MEMCCPY(state->error, errors[ERR_CANT_ALLOC],
++				LINE_DEFAULT, temp);
++			state->running = 0;
++			return;
++		}
++		pclist = state->ucomments;
++		cm     = state->comments;
++		while (cm->start_text != NULL)
++		{
++			uint32_t ucm[MAX(cm->start_size, cm->end_size)];
++			u8_string_to_unicode(ucm, cm->start_text,
++				cm->start_size - 1);
++			pclist->start_text
++				= calloc(cm->start_size, sizeof(*ucm));
++			if (!pclist->start_text)
++			{
++				MEMCCPY(state->error, errors[ERR_CANT_ALLOC],
++					LINE_DEFAULT, temp);
++				state->running = 0;
++				return;
++			}
++			U32_MEMCCPY(pclist->start_text, ucm, cm->start_size,
++				temp32);
++			pclist->start_size = cm->start_size;
++			if (!cm->end_text)
++			{
++				pclist->end_text = NULL;
++				pclist->end_size = -1;
++				goto pclist_skip_copy;
++			}
++			u8_string_to_unicode(ucm, cm->end_text,
++				cm->end_size - 1);
++			pclist->end_text = calloc(cm->end_size, sizeof(*ucm));
++			if (!pclist->end_text)
++			{
++				MEMCCPY(state->error, errors[ERR_CANT_ALLOC],
++					LINE_DEFAULT, temp);
++				state->running = 0;
++				return;
++			}
++			U32_MEMCCPY(pclist->end_text, ucm, cm->end_size, temp32);
++			pclist->end_size = cm->end_size;
++		pclist_skip_copy:
++			cm++;
++			pclist++;
++		}
++		pclist->start_text = NULL;
++		pclist->start_size = -1;
++	}
++	if (!state->uoperators)
++	{
++		struct Operator* op = state->operators;
++		struct Operator32* polist;
++		size_t operator_count = 0;
++		while (op->text != NULL)
++		{
++			operator_count++;
++			op++;
++		}
++		assert(operator_count > 0);
++		state->uoperators
++			= calloc(operator_count + 1, sizeof(struct Operator32));
++		if (!state->uoperators)
++		{
++			MEMCCPY(state->error, errors[ERR_CANT_ALLOC],
++				LINE_DEFAULT, temp);
++			state->running = 0;
++			return;
++		}
++		polist = state->uoperators;
++		op     = state->operators;
++		while (op->text != NULL)
++		{
++			uint32_t uop[op->size];
++			u8_string_to_unicode(uop, op->text, op->size - 1);
++			polist->text = calloc(op->size, sizeof(*uop));
++			if (!polist->text)
++			{
++				MEMCCPY(state->error, errors[ERR_CANT_ALLOC],
++					LINE_DEFAULT, temp);
++				state->running = 0;
++				return;
++			}
++			U32_MEMCCPY(polist->text, uop, op->size, temp32);
++			polist->size = op->size;
++			op++;
++			polist++;
++		}
++		polist->text = NULL;
++		polist->size = -1;
++	}
 +	li	      = 0;
 +	quote	      = 0;
 +	comment_index = -1;
 +recalc_syn_next_line:
 +	curline = state->buffer + li;
 +	free(curline->syntax);
-+	curline->syntax	       = NULL;
-+	curline->syntax_length = 0;
-+	curline->syntax_size   = 0;
++	curline->syntax	     = NULL;
++	curline->syntax_size = 0;
 +	if (li == state->rows_count)
 +		return;
 +
@@ -445,15 +654,8 @@ index f72c9de89..9fba930eb 100644
 +		seen_end_comment = 0;
 +		while (ti < curline->length)
 +		{
-+			size_t ucsize
-+				= 6 * state->comments[comment_index].end_size
-+				+ 1;
-+			uint32_t ucomm[ucsize];
-+			u8_string_to_unicode(ucomm,
-+				state->comments[comment_index].end_text,
-+				ucsize - 1);
-+			seen_end_comment
-+				= u32_starts_with(&curline->text[ti], ucomm);
++			seen_end_comment = u32_starts_with(&curline->text[ti],
++				state->ucomments[comment_index].end_text);
 +			if (seen_end_comment)
 +				break;
 +			ti++;
@@ -482,14 +684,12 @@ index f72c9de89..9fba930eb 100644
 +	}
 +
 +	i	= 0;
-+	comment = state->comments;
++	comment = state->ucomments;
 +	while (comment->start_text != NULL)
 +	{
-+		size_t ucsize = 6 * comment->start_size + 1;
-+		uint32_t ucomm[ucsize];
-+		u8_string_to_unicode(ucomm, comment->start_text, ucsize - 1);
 +		if ((curline->length >= ti + comment->start_size - 1)
-+			&& (u32_starts_with(&curline->text[ti], ucomm)))
++			&& (u32_starts_with(&curline->text[ti],
++				comment->start_text)))
 +		{
 +			assert(comment->start_size > 1);
 +			if (comment->end_text != NULL)
@@ -514,7 +714,7 @@ index f72c9de89..9fba930eb 100644
 +			goto recalc_syn_comment_rest_of_line;
 +		}
 +		i++;
-+		comment = state->comments + i;
++		comment = state->ucomments + i;
 +	}
 +
 +	/* Check for string literals */
@@ -576,19 +776,16 @@ index f72c9de89..9fba930eb 100644
 +
 +	/* Check for keywords */
 +	i	= 0;
-+	keyword = state->keywords;
++	keyword = state->ukeywords;
 +	while (keyword->text != NULL)
 +	{
-+		size_t ucsize = 6 * keyword->size + 1;
-+		uint32_t ucomm[ucsize];
-+		u8_string_to_unicode(ucomm, keyword->text, ucsize - 1);
 +		if ((curline->length >= ti + keyword->size - 1)
 +			&& ((ti == 0)
 +				|| ((ti > 0) && !isalnum(curline->text[ti - 1])
 +					&& curline->text[ti - 1] != L'_'))
 +			&& (!isalnum(curline->text[ti + keyword->size - 1]))
 +			&& (curline->text[ti + keyword->size - 1] != L'_')
-+			&& (u32_starts_with(&curline->text[ti], ucomm)))
++			&& (u32_starts_with(&curline->text[ti], keyword->text)))
 +		{
 +			/* Add new syntax region */
 +			assert(keyword->size > 1);
@@ -605,21 +802,18 @@ index f72c9de89..9fba930eb 100644
 +			goto recalc_syn_next_token;
 +		}
 +		i++;
-+		keyword = state->keywords + i;
++		keyword = state->ukeywords + i;
 +	}
 +
 +	/* Check for operators */
 +	i = 0;
 +	/* clang-format off */
-+	operator = state->operators;
++	operator = state->uoperators;
 +	/* clang-format on */
 +	while (operator->text != NULL)
 +	{
-+		size_t ucsize = 6 * operator->size + 1;
-+		uint32_t ucomm[ucsize];
-+		u8_string_to_unicode(ucomm, operator->text, ucsize - 1);
 +		if ((curline->length >= ti + operator->size - 1)
-+			&& (u32_starts_with(&curline->text[ti], ucomm)))
++			&& (u32_starts_with(&curline->text[ti], operator->text)))
 +		{
 +			/* Add new syntax region */
 +			assert(operator->size> 1);
@@ -637,7 +831,7 @@ index f72c9de89..9fba930eb 100644
 +		}
 +		i++;
 +		/* clang-format off */
-+		operator = state->operators + i;
++		operator = state->uoperators + i;
 +		/* clang-format on */
 +	}
 +
@@ -649,7 +843,7 @@ index f72c9de89..9fba930eb 100644
  redraw_status(struct DrawState* state)
  {
  	assert(state != NULL);
-@@ -791,7 +1226,8 @@ redraw_status(struct DrawState* state)
+@@ -791,7 +1402,8 @@ redraw_status(struct DrawState* state)
   * */
  void
  u32_draw_string(const int x, const int y, const uint16_t fg, const uint16_t bg,
@@ -659,7 +853,7 @@ index f72c9de89..9fba930eb 100644
  	const int p_r, const Alignment align, const int ds, const int fill,
  	const uint32_t* srch, const uint16_t sfg, const uint16_t sbg,
  	const int ssi, const int sei, const uint16_t slfg, const uint16_t slbg)
-@@ -829,6 +1265,8 @@ u32_draw_string(const int x, const int y, const uint16
+@@ -829,6 +1441,8 @@ u32_draw_string(const int x, const int y, const uint16
  
  	int w = 0; /* actual on-screen rune width */
  
@@ -668,7 +862,7 @@ index f72c9de89..9fba930eb 100644
  	const int ts = TAB_SIZE;
  
  	const uint32_t* ps = s; /* pointer to current character in
-@@ -837,11 +1275,23 @@ u32_draw_string(const int x, const int y, const uint16
+@@ -837,11 +1451,23 @@ u32_draw_string(const int x, const int y, const uint16
  	uint16_t cfg = fg; /* current fg */
  	uint16_t cbg = bg; /* current bg */
  
@@ -692,7 +886,7 @@ index f72c9de89..9fba930eb 100644
  	c = mc - p_l - p_r;
  
  	if (align == LEFT)
-@@ -873,7 +1323,12 @@ u32_draw_string(const int x, const int y, const uint16
+@@ -873,7 +1499,12 @@ u32_draw_string(const int x, const int y, const uint16
  	 */
  	while (*ps && sc < c)
  	{
@@ -706,7 +900,7 @@ index f72c9de89..9fba930eb 100644
  		{
  			cfg = slfg;
  			cbg = slbg;
-@@ -890,6 +1345,9 @@ u32_draw_string(const int x, const int y, const uint16
+@@ -890,6 +1521,9 @@ u32_draw_string(const int x, const int y, const uint16
  			}
  		}
  
@@ -716,7 +910,7 @@ index f72c9de89..9fba930eb 100644
  		if (*ps == '\t')
  		{
  			if (ssc < ds)
-@@ -1005,11 +1463,11 @@ u32_draw_string(const int x, const int y, const uint16
+@@ -1005,11 +1639,11 @@ u32_draw_string(const int x, const int y, const uint16
  			}
  			ssc += w;
  		}
@@ -733,7 +927,7 @@ index f72c9de89..9fba930eb 100644
  	}
  
 diff --git a/draw.h b/draw.h
-index f72c9de89..9fba930eb 100644
+index 69b585a99..167fce2e3 100644
 --- a/draw.h
 +++ b/draw.h
 @@ -1,6 +1,7 @@
@@ -744,7 +938,7 @@ index f72c9de89..9fba930eb 100644
  #include <stddef.h>
  #include <sys/types.h>
  #include <termios.h>
-@@ -37,17 +38,58 @@ enum {
+@@ -37,17 +38,74 @@ enum {
  
  typedef enum { LEFT, CENTER, RIGHT } Alignment;
  
@@ -763,6 +957,11 @@ index f72c9de89..9fba930eb 100644
 +	int size;
 +};
 +
++struct Keyword32 {
++	uint32_t* text;
++	int size;
++};
++
 +struct Comment {
 +	char* start_text;
 +	char* end_text;
@@ -770,11 +969,23 @@ index f72c9de89..9fba930eb 100644
 +	int end_size;
 +};
 +
++struct Comment32 {
++	uint32_t* start_text;
++	uint32_t* end_text;
++	int start_size;
++	int end_size;
++};
++
 +struct Operator {
 +	char* text;
 +	int size;
 +};
 +
++struct Operator32 {
++	uint32_t* text;
++	int size;
++};
++
  struct Coord {
  	int col;
  	int row;
@@ -798,12 +1009,11 @@ index f72c9de89..9fba930eb 100644
  	uint16_t fg;
  	uint16_t bg;
 +	struct SyntaxRegion* syntax;
-+	ssize_t syntax_length;
 +	ssize_t syntax_size;
  };
  
  struct DrawState {
-@@ -87,12 +129,17 @@ struct DrawState {
+@@ -87,12 +145,20 @@ struct DrawState {
  	int dirty;
  	int new_file;
  	int recalculate_size;
@@ -818,10 +1028,13 @@ index f72c9de89..9fba930eb 100644
 +	struct Keyword* keywords;
 +	struct Comment* comments;
 +	struct Operator* operators;
++	struct Keyword32* ukeywords; /* Converted to 32-bit Unicode */
++	struct Comment32* ucomments;
++	struct Operator32* uoperators;
  };
  
  struct Key {
-@@ -109,6 +156,13 @@ struct StatusSegment {
+@@ -109,6 +175,13 @@ struct StatusSegment {
  	Alignment alignment;
  };
  
@@ -835,7 +1048,7 @@ index f72c9de89..9fba930eb 100644
  void assign_coord(struct Coord* dest, const struct Coord src);
  ssize_t buffer_length(uint32_t* buffer, const ssize_t max_index);
  int calculate_help_width(const struct DrawState* state);
-@@ -125,7 +179,8 @@ void draw_rect(const int startx, const int endx, const
+@@ -125,7 +198,8 @@ void draw_rect(const int startx, const int endx, const
  int draw_screen(const struct DrawState* state);
  int draw_status(const struct DrawState* state);
  int draw_string(const int x, const int y, const uint16_t fg, const uint16_t bg,
@@ -845,7 +1058,7 @@ index f72c9de89..9fba930eb 100644
  	const int padding_left, const int padding_right, const Alignment align,
  	const int ds, const int fill, const uint32_t* srch, const uint16_t sfg,
  	const uint16_t sbg, const int ssi, const int sei, const uint16_t slfg,
-@@ -148,10 +203,12 @@ void init_drawstate(struct DrawState* state, char* err
+@@ -148,10 +222,12 @@ void init_drawstate(struct DrawState* state, char* err
  	const int skip_filenames_init);
  void recalculate_line_numbers(struct DrawState* state);
  void recalculate_size(struct DrawState* state);
@@ -864,7 +1077,7 @@ index f72c9de89..9fba930eb 100644
 +	const uint16_t sbg, const int ssi, const int sei, const uint16_t slfg,
 +	const uint16_t slbg);
 diff --git a/sled.c b/sled.c
-index f72c9de89..9fba930eb 100644
+index 69b585a99..167fce2e3 100644
 --- a/sled.c
 +++ b/sled.c
 @@ -67,6 +67,18 @@ extern const char* prompt_reread_dirty;