Add support for manually setting subpixel hinting on outputs.

Many laptop screens report unknown subpixel order. Allow users to manually set subpixel hinting to work around this.

Addresses https://github.com/swaywm/sway/issues/3163

common/util.c

@@ -4,6 +4,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <strings.h>
+#include <wayland-server-protocol.h>
 #include "log.h"
 #include "util.h"
 
@@ -54,3 +55,23 @@ float parse_float(const char *value) {
 	}
 	return flt;
 }
+
+
+const char *sway_wl_output_subpixel_to_string(enum wl_output_subpixel subpixel) {
+	switch (subpixel) {
+	case WL_OUTPUT_SUBPIXEL_UNKNOWN:
+		return "unknown";
+	case WL_OUTPUT_SUBPIXEL_NONE:
+		return "none";
+	case WL_OUTPUT_SUBPIXEL_HORIZONTAL_RGB:
+		return "rgb";
+	case WL_OUTPUT_SUBPIXEL_HORIZONTAL_BGR:
+		return "bgr";
+	case WL_OUTPUT_SUBPIXEL_VERTICAL_RGB:
+		return "vrgb";
+	case WL_OUTPUT_SUBPIXEL_VERTICAL_BGR:
+		return "vbgr";
+	}
+	sway_assert(false, "Unknown value for wl_output_subpixel.");
+	return NULL;
+}

include/sway/commands.h

@@ -261,6 +261,7 @@ sway_cmd output_cmd_enable;
 sway_cmd output_cmd_mode;
 sway_cmd output_cmd_position;
 sway_cmd output_cmd_scale;
+sway_cmd output_cmd_subpixel;
 sway_cmd output_cmd_transform;
 
 sway_cmd seat_cmd_attach;

include/sway/config.h

@@ -184,6 +184,7 @@ struct output_config {
 	int x, y;
 	float scale;
 	int32_t transform;
+	enum wl_output_subpixel subpixel;
 
 	char *background;
 	char *background_option;

include/sway/output.h

@@ -31,6 +31,7 @@ struct sway_output {
 
 	int lx, ly; // layout coords
 	int width, height; // transformed buffer size
+	enum wl_output_subpixel detected_subpixel;
 
 	bool enabled, configured;
 	list_t *workspaces;

include/util.h

@@ -3,6 +3,7 @@
 
 #include <stdint.h>
 #include <stdbool.h>
+#include <wayland-server-protocol.h>
 
 /**
  * Wrap i into the range [0, max[
@@ -29,4 +30,6 @@ bool parse_boolean(const char *boolean, bool current);
  */
 float parse_float(const char *value);
 
+const char *sway_wl_output_subpixel_to_string(enum wl_output_subpixel subpixel);
+
 #endif

sway/commands/output.c

@@ -18,6 +18,7 @@ static struct cmd_handler output_handlers[] = {
 	{ "res", output_cmd_mode },
 	{ "resolution", output_cmd_mode },
 	{ "scale", output_cmd_scale },
+	{ "subpixel", output_cmd_subpixel },
 	{ "transform", output_cmd_transform },
 };
 

sway/commands/output/subpixel.c

@@ -0,0 +1,36 @@
+#include <string.h>
+#include "log.h"
+#include "sway/commands.h"
+#include "sway/config.h"
+#include "sway/output.h"
+
+struct cmd_results *output_cmd_subpixel(int argc, char **argv) {
+	if (!config->handler_context.output_config) {
+		return cmd_results_new(CMD_FAILURE, "Missing output config");
+	}
+	if (!argc) {
+		return cmd_results_new(CMD_INVALID, "Missing subpixel argument.");
+	}
+	enum wl_output_subpixel subpixel;
+
+	if (strcmp(*argv, "rgb") == 0) {
+		subpixel = WL_OUTPUT_SUBPIXEL_HORIZONTAL_RGB;
+	} else if (strcmp(*argv, "bgr") == 0) {
+		subpixel = WL_OUTPUT_SUBPIXEL_HORIZONTAL_BGR;
+	} else if (strcmp(*argv, "vrgb") == 0) {
+		subpixel = WL_OUTPUT_SUBPIXEL_VERTICAL_RGB;
+	} else if (strcmp(*argv, "vbgr") == 0) {
+		subpixel = WL_OUTPUT_SUBPIXEL_VERTICAL_BGR;
+	} else if (strcmp(*argv, "none") == 0) {
+		subpixel = WL_OUTPUT_SUBPIXEL_NONE;
+	} else {
+		return cmd_results_new(CMD_INVALID, "Invalid output subpixel.");
+	}
+
+	struct output_config *oc = config->handler_context.output_config;
+	config->handler_context.leftovers.argc = argc - 1;
+	config->handler_context.leftovers.argv = argv + 1;
+
+	oc->subpixel = subpixel;
+	return NULL;
+}

sway/config/output.c

@@ -8,10 +8,11 @@
 #include <unistd.h>
 #include <wlr/types/wlr_output_layout.h>
 #include <wlr/types/wlr_output.h>
-#include "log.h"
 #include "sway/config.h"
 #include "sway/output.h"
 #include "sway/tree/root.h"
+#include "log.h"
+#include "util.h"
 
 int output_name_cmp(const void *item, const void *data) {
 	const struct output_config *output = item;
@@ -43,6 +44,7 @@ struct output_config *new_output_config(const char *name) {
 	oc->x = oc->y = -1;
 	oc->scale = -1;
 	oc->transform = -1;
+	oc->subpixel = WL_OUTPUT_SUBPIXEL_UNKNOWN;
 	return oc;
 }
 
@@ -65,6 +67,9 @@ void merge_output_config(struct output_config *dst, struct output_config *src) {
 	if (src->scale != -1) {
 		dst->scale = src->scale;
 	}
+	if (src->subpixel != WL_OUTPUT_SUBPIXEL_UNKNOWN) {
+		dst->subpixel = src->subpixel;
+	}
 	if (src->refresh_rate != -1) {
 		dst->refresh_rate = src->refresh_rate;
 	}
@@ -187,10 +192,10 @@ struct output_config *store_output_config(struct output_config *oc) {
 	}
 
 	sway_log(SWAY_DEBUG, "Config stored for output %s (enabled: %d) (%dx%d@%fHz "
-		"position %d,%d scale %f transform %d) (bg %s %s) (dpms %d)",
+		"position %d,%d scale %f subpixel %s transform %d) (bg %s %s) (dpms %d)",
 		oc->name, oc->enabled, oc->width, oc->height, oc->refresh_rate,
-		oc->x, oc->y, oc->scale, oc->transform, oc->background,
+		oc->x, oc->y, oc->scale, sway_wl_output_subpixel_to_string(oc->subpixel),
-		oc->background_option, oc->dpms_state);
+		oc->transform, oc->background, oc->background_option, oc->dpms_state);
 
 	return oc;
 }
@@ -363,6 +368,14 @@ bool apply_output_config(struct output_config *oc, struct sway_output *output) {
 		sway_log(SWAY_DEBUG, "Set %s scale to %f", oc->name, oc->scale);
 		wlr_output_set_scale(wlr_output, oc->scale);
 	}
+
+	if (oc && (oc->subpixel != WL_OUTPUT_SUBPIXEL_UNKNOWN || config->reloading)) {
+		sway_log(SWAY_DEBUG, "Set %s subpixel to %s", oc->name,
+			sway_wl_output_subpixel_to_string(oc->subpixel));
+		wlr_output_set_subpixel(wlr_output, oc->subpixel);
+		output_damage_whole(output);
+	}
+
 	if (oc && oc->transform >= 0) {
 		sway_log(SWAY_DEBUG, "Set %s transform to %d", oc->name, oc->transform);
 		wlr_output_set_transform(wlr_output, oc->transform);
@@ -424,6 +437,8 @@ static void default_output_config(struct output_config *oc,
 	}
 	oc->x = oc->y = -1;
 	oc->scale = 1;
+	struct sway_output *output = wlr_output->data;
+	oc->subpixel = output->detected_subpixel;
 	oc->transform = WL_OUTPUT_TRANSFORM_NORMAL;
 	oc->dpms_state = DPMS_ON;
 }

sway/ipc-server.c

@@ -644,6 +644,8 @@ void ipc_client_handle_command(struct ipc_client *client) {
 			json_object_object_add(output_json, "focused",
 				json_object_new_boolean(focused));
 
+			const char *subpixel = sway_wl_output_subpixel_to_string(output->wlr_output->subpixel);
+			json_object_object_add(output_json, "subpixel_hinting", json_object_new_string(subpixel));
 			json_object_array_add(outputs, output_json);
 		}
 		struct sway_output *output;

sway/meson.build

@@ -169,6 +169,7 @@ sway_sources = files(
 	'commands/output/mode.c',
 	'commands/output/position.c',
 	'commands/output/scale.c',
+	'commands/output/subpixel.c',
 	'commands/output/transform.c',
 
 	'tree/arrange.c',

sway/sway-ipc.7.scd

@@ -211,6 +211,9 @@ following properties:
 |- scale
 :  float
 :  The scale currently in use on the output or _-1_ for disabled outputs
+|- subpixel_hinting
+:  string
+:  The subpixel hinting current in use on the output. This can be _rgb_, _bgr_, _vrgb_, _vbgr_, or _none_
 |- transform
 :  string
 :  The transform currently in use for the output. This can be _normal_, _90_,
@@ -242,6 +245,7 @@ following properties:
 		"active": true,
 		"primary": false,
 		"scale": 1.0,
+		"subpixel_hinting": "rgb",
 		"transform": "normal",
 		"current_workspace": "1",
 		"modes": [

sway/sway-output.5.scd

@@ -64,6 +64,13 @@ must be separated by one space. For example:
 	applications to taste. HiDPI isn't supported with Xwayland clients (windows
 	will blur).
 
+*output* <name> subpixel rgb|bgr|vrgb|vbgr|none
+	Manually sets the subpixel hinting for the specified output. This value is
+	usually auto-detected, but some displays may misreport their subpixel
+	geometry. Using the correct subpixel hinting allows for sharper text.
+	Incorrect values will result in blurrier text. When changing this via
+	*swaymsg*, some applications may need to be restarted to use the new value.
+
 *output* <name> background|bg <file> <mode> [<fallback_color>]
 	Sets the wallpaper for the given output to the specified file, using the
 	given scaling mode (one of "stretch", "fill", "fit", "center", "tile"). If

sway/tree/output.c

@@ -92,6 +92,7 @@ struct sway_output *output_create(struct wlr_output *wlr_output) {
 	node_init(&output->node, N_OUTPUT, output);
 	output->wlr_output = wlr_output;
 	wlr_output->data = output;
+	output->detected_subpixel = wlr_output->subpixel;
 
 	wl_signal_init(&output->events.destroy);
 

swaymsg/main.c

@@ -186,11 +186,12 @@ static void pretty_print_output(json_object *o) {
 	json_object_object_get_ex(o, "focused", &focused);
 	json_object_object_get_ex(o, "active", &active);
 	json_object_object_get_ex(o, "current_workspace", &ws);
-	json_object *make, *model, *serial, *scale, *transform;
+	json_object *make, *model, *serial, *scale, *subpixel, *transform;
 	json_object_object_get_ex(o, "make", &make);
 	json_object_object_get_ex(o, "model", &model);
 	json_object_object_get_ex(o, "serial", &serial);
 	json_object_object_get_ex(o, "scale", &scale);
+	json_object_object_get_ex(o, "subpixel_hinting", &subpixel);
 	json_object_object_get_ex(o, "transform", &transform);
 	json_object *x, *y;
 	json_object_object_get_ex(rect, "x", &x);
@@ -209,6 +210,7 @@ static void pretty_print_output(json_object *o) {
 			"  Current mode: %dx%d @ %f Hz\n"
 			"  Position: %d,%d\n"
 			"  Scale factor: %f\n"
+			"  Subpixel hinting: %s\n"
 			"  Transform: %s\n"
 			"  Workspace: %s\n",
 			json_object_get_string(name),
@@ -221,6 +223,7 @@ static void pretty_print_output(json_object *o) {
 			(float)json_object_get_int(refresh) / 1000,
 			json_object_get_int(x), json_object_get_int(y),
 			json_object_get_double(scale),
+			json_object_get_string(subpixel),
 			json_object_get_string(transform),
 			json_object_get_string(ws)
 		);