Add API to query candidate planes

This can be used to implement Wayland's linux-dmabuf-unstable-v1's
feedback mechanism.

Closes: https://gitlab.freedesktop.org/emersion/libliftoff/-/issues/70

alloc.c

@@ -439,7 +439,7 @@ output_choose_layers(struct liftoff_output *output, struct alloc_result *result,
 		    step->log_prefix, plane->id, step->plane_idx + 1, result->planes_len);
 
 	liftoff_list_for_each(layer, &output->layers, link) {
-		if (layer->plane != NULL || layer->force_composition) {
+		if (layer->plane != NULL) {
 			continue;
 		}
 		if (!layer_is_visible(layer)) {
@@ -461,6 +461,18 @@ output_choose_layers(struct liftoff_output *output, struct alloc_result *result,
 			return ret;
 		}
 
+		layer_add_candidate_plane(layer, plane);
+
+		/* If composition is forced, wait until after the
+		 * layer_add_candidate_plane() call to reject the plane: we want
+		 * to return a meaningful list of candidate planes so that the
+		 * API user has the opportunity to re-allocate its buffers with
+		 * scanout-capable ones. */
+		if (layer->force_composition) {
+			drmModeAtomicSetCursor(result->req, cursor);
+			continue;
+		}
+
 		ret = device_test_commit(device, result->req, result->flags);
 		if (ret == 0) {
 			liftoff_log(LIFTOFF_DEBUG,
@@ -743,6 +755,11 @@ liftoff_output_apply(struct liftoff_output *output, drmModeAtomicReq *req,
 	}
 	log_no_reuse(output);
 
+	/* Reset layers' candidate planes */
+	liftoff_list_for_each(layer, &output->layers, link) {
+		layer_reset_candidate_planes(layer);
+	}
+
 	device->test_commit_counter = 0;
 	output_log_layers(output);
 

device.c

@@ -10,6 +10,7 @@ liftoff_device_create(int drm_fd)
 {
 	struct liftoff_device *device;
 	drmModeRes *drm_res;
+	drmModePlaneRes *drm_plane_res;
 
 	device = calloc(1, sizeof(*device));
 	if (device == NULL) {
@@ -47,6 +48,15 @@ liftoff_device_create(int drm_fd)
 
 	drmModeFreeResources(drm_res);
 
+	drm_plane_res = drmModeGetPlaneResources(device->drm_fd);
+	if (drm_plane_res == NULL) {
+		liftoff_log_errno(LIFTOFF_ERROR, "drmModeGetPlaneResources");
+		liftoff_device_destroy(device);
+		return NULL;
+	}
+	device->planes_cap = drm_plane_res->count_planes;
+	drmModeFreePlaneResources(drm_plane_res);
+
 	return device;
 }
 

include/libliftoff.h

@@ -179,6 +179,21 @@ liftoff_layer_needs_composition(struct liftoff_layer *layer);
 struct liftoff_plane *
 liftoff_layer_get_plane(struct liftoff_layer *layer);
 
+/**
+ * Check whether a plane is a candidate for this layer.
+ *
+ * A plane is a candidate if it could potentially be used for the layer with
+ * a buffer with the same size. The buffer may need to be re-allocated with
+ * formats and modifiers accepted by the plane.
+ *
+ * This can be used to implemented a feedback loop: if a layer isn't mapped to
+ * a plane, loop over the candidate planes, and re-allocate the layer's FB
+ * according to the IN_FORMATS property.
+ */
+bool
+liftoff_layer_is_candidate_plane(struct liftoff_layer *layer,
+				 struct liftoff_plane *plane);
+
 enum liftoff_log_priority {
 	LIFTOFF_SILENT,
 	LIFTOFF_ERROR,

include/private.h

@@ -18,6 +18,8 @@ struct liftoff_device {
 	uint32_t *crtcs;
 	size_t crtcs_len;
 
+	size_t planes_cap; /* max number of planes */
+
 	int page_flip_counter;
 	int test_commit_counter;
 };
@@ -48,6 +50,9 @@ struct liftoff_layer {
 
 	struct liftoff_plane *plane;
 
+	/* Array of plane IDs with a length of liftoff_device.planes_cap */
+	uint32_t *candidate_planes;
+
 	int current_priority, pending_priority;
 	/* prop added or force_composition changed */
 	bool changed;
@@ -105,6 +110,13 @@ layer_update_priority(struct liftoff_layer *layer, bool make_current);
 bool
 layer_has_fb(struct liftoff_layer *layer);
 
+void
+layer_add_candidate_plane(struct liftoff_layer *layer,
+			  struct liftoff_plane *plane);
+
+void
+layer_reset_candidate_planes(struct liftoff_layer *layer);
+
 bool
 layer_is_visible(struct liftoff_layer *layer);
 

layer.c

@@ -1,7 +1,9 @@
+#include <assert.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <string.h>
 #include <xf86drm.h>
+#include <sys/types.h>
 #include "private.h"
 
 struct liftoff_layer *
@@ -15,6 +17,13 @@ liftoff_layer_create(struct liftoff_output *output)
 		return NULL;
 	}
 	layer->output = output;
+	layer->candidate_planes = calloc(sizeof(layer->candidate_planes[0]),
+					 output->device->planes_cap);
+	if (layer->candidate_planes == NULL) {
+		liftoff_log_errno(LIFTOFF_ERROR, "calloc");
+		free(layer);
+		return NULL;
+	}
 	liftoff_list_insert(output->layers.prev, &layer->link);
 	output->layers_changed = true;
 	return layer;
@@ -35,6 +44,7 @@ liftoff_layer_destroy(struct liftoff_layer *layer)
 		layer->output->composition_layer = NULL;
 	}
 	free(layer->props);
+	free(layer->candidate_planes);
 	liftoff_list_remove(&layer->link);
 	free(layer);
 }
@@ -296,3 +306,45 @@ layer_cache_fb_info(struct liftoff_layer *layer)
 	drmModeFreeFB2(fb_info);
 	return 0;
 }
+
+bool
+liftoff_layer_is_candidate_plane(struct liftoff_layer *layer,
+				 struct liftoff_plane *plane)
+{
+	size_t i;
+
+	for (i = 0; i < layer->output->device->planes_cap; i++) {
+		if (layer->candidate_planes[i] == plane->id) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+void
+layer_add_candidate_plane(struct liftoff_layer *layer,
+			  struct liftoff_plane *plane)
+{
+	size_t i;
+	ssize_t empty_slot = -1;
+
+	for (i = 0; i < layer->output->device->planes_cap; i++) {
+		if (layer->candidate_planes[i] == plane->id) {
+			return;
+		}
+		if (empty_slot < 0 && layer->candidate_planes[i] == 0) {
+			empty_slot = i;
+		}
+	}
+
+	assert(empty_slot >= 0);
+	layer->candidate_planes[empty_slot] = plane->id;
+}
+
+void
+layer_reset_candidate_planes(struct liftoff_layer *layer)
+{
+	memset(layer->candidate_planes, 0,
+	       sizeof(layer->candidate_planes[0]) * layer->output->device->planes_cap);
+}