Take disabled layers into account in non_composition_layers_len

Disabled layers (with FB_ID = 0) don't need a plane. Fix the
non_composition_layers_len count in this case.

Closes: https://github.com/emersion/libliftoff/issues/49

alloc.c

@@ -343,6 +343,15 @@ bool check_alloc_valid(struct alloc_result *result, struct alloc_step *step)
 	return true;
 }
 
+static bool layer_is_visible(struct liftoff_layer *layer)
+{
+	if (layer->force_composition) {
+		return true;
+	} else {
+		return layer_has_fb(layer);
+	}
+}
+
 bool output_choose_layers(struct liftoff_output *output,
 			  struct alloc_result *result, struct alloc_step *step)
 {
@@ -398,8 +407,8 @@ bool output_choose_layers(struct liftoff_output *output,
 		if (layer->plane != NULL || layer->force_composition) {
 			continue;
 		}
-		if (!layer_has_fb(layer)) {
+		if (!layer_is_visible(layer)) {
-			continue; /* no FB set, nothing to display */
+			continue;
 		}
 		if (!check_layer_plane_compatible(step, layer, plane)) {
 			continue;
@@ -582,6 +591,22 @@ static void log_no_reuse(struct liftoff_output *output)
 	}
 }
 
+static size_t non_composition_layers_length(struct liftoff_output *output)
+{
+	struct liftoff_layer *layer;
+	size_t n;
+
+	n = 0;
+	liftoff_list_for_each(layer, &output->layers, link) {
+		if (layer_is_visible(layer) &&
+		    output->composition_layer != layer) {
+			n++;
+		}
+	}
+
+	return n;
+}
+
 bool liftoff_output_apply(struct liftoff_output *output, drmModeAtomicReq *req)
 {
 	struct liftoff_device *device;
@@ -644,10 +669,7 @@ bool liftoff_output_apply(struct liftoff_output *output, drmModeAtomicReq *req)
 	memset(result.best, 0, result.planes_len * sizeof(*result.best));
 	result.has_composition_layer = output->composition_layer != NULL;
 	result.non_composition_layers_len =
-		liftoff_list_length(&output->layers);
+		non_composition_layers_length(output);
-	if (output->composition_layer != NULL) {
-		result.non_composition_layers_len--;
-	}
 	step.plane_link = device->planes.next;
 	step.plane_idx = 0;
 	step.score = 0;

test/meson.build

@@ -24,8 +24,9 @@ bench_exe = executable(
 tests = {
 	'alloc': [
 		'basic',
-		'no-props',
+		'no-props-fail',
-		'zero-fb-id',
+		'zero-fb-id-fail',
+		'composition-zero-fb-id',
 		'empty',
 		'simple-1x',
 		'simple-1x-fail',

test/test_alloc.c

@@ -720,7 +720,7 @@ static void test_basic(void)
 
 /* Checks that the library doesn't allocate a plane for a layer without a
  * non-zero FB_ID set. */
-static void test_no_fb(bool zero_fb_id)
+static void test_no_fb_fail(bool zero_fb_id)
 {
 	struct liftoff_mock_plane *mock_plane;
 	int drm_fd;
@@ -757,6 +757,50 @@ static void test_no_fb(bool zero_fb_id)
 	close(drm_fd);
 }
 
+/* Checks that the library doesn't fallback to composition when a layer doesn't
+ * have a FB. */
+static void test_composition_zero_fb(void)
+{
+	struct liftoff_mock_plane *mock_plane;
+	int drm_fd;
+	struct liftoff_device *device;
+	struct liftoff_output *output;
+	struct liftoff_layer *composition_layer, *layer_with_fb,
+			     *layer_without_fb;
+	drmModeAtomicReq *req;
+	bool ok;
+	int ret;
+
+	mock_plane = liftoff_mock_drm_create_plane(DRM_PLANE_TYPE_PRIMARY);
+
+	drm_fd = liftoff_mock_drm_open();
+	device = liftoff_device_create(drm_fd);
+	assert(device != NULL);
+
+	output = liftoff_output_create(device, liftoff_mock_drm_crtc_id);
+	composition_layer = add_layer(output, 0, 0, 1920, 1080);
+	layer_with_fb = add_layer(output, 0, 0, 1920, 1080);
+	layer_without_fb = liftoff_layer_create(output);
+	(void)layer_with_fb;
+
+	liftoff_output_set_composition_layer(output, composition_layer);
+
+	liftoff_mock_plane_add_compatible_layer(mock_plane, composition_layer);
+	liftoff_mock_plane_add_compatible_layer(mock_plane, layer_without_fb);
+	liftoff_mock_plane_add_compatible_layer(mock_plane, layer_with_fb);
+
+	req = drmModeAtomicAlloc();
+	ok = liftoff_output_apply(output, req);
+	assert(ok);
+	ret = drmModeAtomicCommit(drm_fd, req, 0, NULL);
+	assert(ret == 0);
+	assert(liftoff_mock_plane_get_layer(mock_plane) == layer_with_fb);
+	drmModeAtomicFree(req);
+
+	liftoff_device_destroy(device);
+	close(drm_fd);
+}
+
 int main(int argc, char *argv[]) {
 	const char *test_name;
 	size_t i;
@@ -772,11 +816,14 @@ int main(int argc, char *argv[]) {
 	if (strcmp(test_name, "basic") == 0) {
 		test_basic();
 		return 0;
-	} else if (strcmp(test_name, "no-props") == 0) {
+	} else if (strcmp(test_name, "no-props-fail") == 0) {
-		test_no_fb(false);
+		test_no_fb_fail(false);
 		return 0;
-	} else if (strcmp(test_name, "zero-fb-id") == 0) {
+	} else if (strcmp(test_name, "zero-fb-id-fail") == 0) {
-		test_no_fb(true);
+		test_no_fb_fail(true);
+		return 0;
+	} else if (strcmp(test_name, "composition-zero-fb-id") == 0) {
+		test_composition_zero_fb();
 		return 0;
 	}