Add a way to force FB composition

Closes: https://github.com/emersion/libliftoff/issues/17

alloc.c

@@ -415,7 +415,7 @@ bool output_choose_layers(struct liftoff_output *output,
 		    plane->id, step->plane_idx + 1, result->planes_len);
 
 	liftoff_list_for_each(layer, &output->layers, link) {
-		if (layer->plane != NULL) {
+		if (layer->plane != NULL || layer->force_composition) {
 			continue;
 		}
 		if (!check_layer_plane_compatible(step, layer, plane)) {
@@ -496,6 +496,9 @@ static bool layer_needs_realloc(struct liftoff_layer *layer)
 			continue;
 		}
 		if (strcmp(prop->name, "FB_ID") == 0) {
+			if (layer->force_composition) {
+				return true;
+			}
 			/* TODO: check format/modifier is the same. Check
 			 * previous/next value isn't zero. */
 			continue;

include/libliftoff.h

@@ -46,11 +46,22 @@ void liftoff_output_set_composition_layer(struct liftoff_output *output,
 struct liftoff_layer *liftoff_layer_create(struct liftoff_output *output);
 void liftoff_layer_destroy(struct liftoff_layer *layer);
 /**
- * Set a property on the layer. Any plane property can be set. If none of the
- * planes support the property, the layer won't be mapped to any plane.
+ * Set a property on the layer. Any plane property can be set (except CRTC_ID).
+ * If none of the planes support the property, the layer won't be mapped to any
+ * plane.
+ *
+ * Setting a zero FB_ID disables the layer.
  */
 void liftoff_layer_set_property(struct liftoff_layer *layer, const char *name,
 				uint64_t value);
+/**
+ * Force composition on this layer. This unsets any previous FB_ID value. To
+ * switch back to direct scan-out, set FB_ID again.
+ *
+ * This can be used when no KMS FB ID is available for this layer but it still
+ * needs to be displayed (e.g. the buffer cannot be imported in KMS).
+ */
+void liftoff_layer_set_fb_composited(struct liftoff_layer *layer);
 /**
  * Retrieve the plane mapped to this layer. Zero is returned if no plane is
  * mapped.

include/private.h

@@ -41,6 +41,8 @@ struct liftoff_layer {
 	struct liftoff_layer_property *props;
 	size_t props_len;
 
+	bool force_composition; /* FB needs to be composited */
+
 	struct liftoff_plane *plane;
 
 	int current_priority, pending_priority;

layer.c

@@ -78,6 +78,26 @@ void liftoff_layer_set_property(struct liftoff_layer *layer, const char *name,
 	}
 
 	prop->value = value;
+
+	if (strcmp(name, "FB_ID") == 0) {
+		layer->force_composition = false;
+		prop->changed = true;
+	}
+}
+
+void liftoff_layer_set_fb_composited(struct liftoff_layer *layer)
+{
+	struct liftoff_layer_property *prop;
+
+	if (layer->force_composition) {
+		return;
+	}
+
+	liftoff_layer_set_property(layer, "FB_ID", 0);
+	prop = layer_get_property(layer, "FB_ID");
+	prop->changed = true;
+
+	layer->force_composition = true;
 }
 
 uint32_t liftoff_layer_get_plane_id(struct liftoff_layer *layer)

test/meson.build

@@ -41,6 +41,7 @@ tests = {
 		'composition-3x',
 		'composition-3x-fail',
 		'composition-3x-partial',
+		'composition-3x-force',
 	],
 	'dynamic': [
 		'same',

test/test_alloc.c

@@ -34,6 +34,7 @@ struct test_layer {
 	int x, y, width, height;
 	int zpos; /* zero means unset */
 	bool composition;
+	bool force_composited;
 	struct test_plane *compat[64];
 	struct test_plane *result;
 };
@@ -553,6 +554,44 @@ static struct test_case tests[] = {
 			},
 		},
 	},
+	{
+		.name = "composition-3x-force",
+		/* Layers at zpos=1 and zpos=2 could be put on a plane, but
+		 * FB composition is forced on the zpos=2 one. As a result, only
+		 * the layer at zpos=3 can be put into a plane. */
+		.layers = {
+			{
+				.width = 1920,
+				.height = 1080,
+				.zpos = 1,
+				.composition = true,
+				.compat = { PRIMARY_PLANE },
+				.result = PRIMARY_PLANE,
+			},
+			{
+				.width = 1920,
+				.height = 1080,
+				.zpos = 1,
+				.compat = { PRIMARY_PLANE },
+				.result = NULL,
+			},
+			{
+				.width = 100,
+				.height = 100,
+				.zpos = 2,
+				.force_composited = true,
+				.compat = FIRST_2_SECONDARY_PLANES,
+				.result = NULL,
+			},
+			{
+				.width = 100,
+				.height = 100,
+				.zpos = 3,
+				.compat = { CURSOR_PLANE },
+				.result = CURSOR_PLANE,
+			},
+		},
+	},
 };
 
 static void run_test(struct test_layer *test_layers)
@@ -590,6 +629,9 @@ static void run_test(struct test_layer *test_layers)
 		if (test_layer->composition) {
 			liftoff_output_set_composition_layer(output, layers[i]);
 		}
+		if (test_layer->force_composited) {
+			liftoff_layer_set_fb_composited(layers[i]);
+		}
 		for (j = 0; test_layer->compat[j] != NULL; j++) {
 			mock_plane = mock_planes[test_layer->compat[j] -
 						 test_setup];