Consider intersections when deciding reallocation

Previously, we would reallocate when any CRTC_* property changes. This
leads to unnecessary reallocations when a plane moves or resizes. Cursor
movements would reallocate on every update, for example.

What we really care about is if a layer's intersection status with
another plane changes, so do that.

alloc.c

@@ -596,10 +596,37 @@ fb_info_needs_realloc(const drmModeFB2 *a, const drmModeFB2 *b)
 }
 
 static bool
-layer_needs_realloc(struct liftoff_layer *layer)
+layer_intersection_changed(struct liftoff_layer *this,
+			   struct liftoff_output *output)
+{
+	struct liftoff_layer *other;
+	struct liftoff_rect this_cur, this_prev, other_cur, other_prev;
+
+	layer_get_rect(this, &this_cur);
+	layer_get_prev_rect(this, &this_prev);
+	liftoff_list_for_each(other, &output->layers, link) {
+		if (this == other) {
+			continue;
+		}
+
+		layer_get_rect(other, &other_cur);
+		layer_get_prev_rect(other, &other_prev);
+
+		if (rect_intersects(&this_cur, &other_cur) !=
+		    rect_intersects(&this_prev, &other_prev)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static bool
+layer_needs_realloc(struct liftoff_layer *layer, struct liftoff_output *output)
 {
-	size_t i;
 	struct liftoff_layer_property *prop;
+	bool check_crtc_intersect = false;
+	size_t i;
 
 	if (layer->changed) {
 		liftoff_log(LIFTOFF_DEBUG, "Cannot re-use previous allocation: "
@@ -664,13 +691,27 @@ layer_needs_realloc(struct liftoff_layer *layer)
 			continue;
 		}
 
-		/* TODO: if CRTC_{X,Y,W,H} changed but intersection with other
+		/* If CRTC_* changed, check for intersection later */
-		 * layers hasn't changed, don't realloc */
+		if (strcmp(prop->name, "CRTC_X") == 0 ||
+		    strcmp(prop->name, "CRTC_Y") == 0 ||
+		    strcmp(prop->name, "CRTC_W") == 0 ||
+		    strcmp(prop->name, "CRTC_H") == 0) {
+			check_crtc_intersect = true;
+			continue;
+		}
+
 		liftoff_log(LIFTOFF_DEBUG, "Cannot re-use previous allocation: "
 			    "property \"%s\" changed", prop->name);
 		return true;
 	}
 
+	if (check_crtc_intersect &&
+	    layer_intersection_changed(layer, output)) {
+		liftoff_log(LIFTOFF_DEBUG, "Cannot re-use previous allocation: "
+			    "intersection with other layer(s) changed");
+		return true;
+	}
+
 	return false;
 }
 
@@ -787,7 +828,7 @@ reuse_previous_alloc(struct liftoff_output *output, drmModeAtomicReq *req,
 	}
 
 	liftoff_list_for_each(layer, &output->layers, link) {
-		if (layer_needs_realloc(layer)) {
+		if (layer_needs_realloc(layer, output)) {
 			return -EINVAL;
 		}
 	}

include/private.h

@@ -93,6 +93,12 @@ layer_get_property(struct liftoff_layer *layer, const char *name);
 void
 layer_get_rect(struct liftoff_layer *layer, struct liftoff_rect *rect);
 
+void
+layer_get_prev_rect(struct liftoff_layer *layer, struct liftoff_rect *rect);
+
+bool
+rect_intersects(struct liftoff_rect *a, struct liftoff_rect *b);
+
 bool
 layer_intersects(struct liftoff_layer *a, struct liftoff_layer *b);
 

layer.c

@@ -167,6 +167,29 @@ layer_get_rect(struct liftoff_layer *layer, struct liftoff_rect *rect)
 	rect->height = h_prop != NULL ? h_prop->value : 0;
 }
 
+void
+layer_get_prev_rect(struct liftoff_layer *layer, struct liftoff_rect *rect)
+{
+	struct liftoff_layer_property *x_prop, *y_prop, *w_prop, *h_prop;
+
+	x_prop = layer_get_property(layer, "CRTC_X");
+	y_prop = layer_get_property(layer, "CRTC_Y");
+	w_prop = layer_get_property(layer, "CRTC_W");
+	h_prop = layer_get_property(layer, "CRTC_H");
+
+	rect->x = x_prop != NULL ? x_prop->prev_value : 0;
+	rect->y = y_prop != NULL ? y_prop->prev_value : 0;
+	rect->width = w_prop != NULL ? w_prop->prev_value : 0;
+	rect->height = h_prop != NULL ? h_prop -> prev_value : 0;
+}
+
+bool
+rect_intersects(struct liftoff_rect *ra, struct liftoff_rect *rb)
+{
+	return ra->x < rb->x + rb->width && ra->y < rb->y + rb->height &&
+	       ra->x + ra->width > rb->x && ra->y + ra->height > rb->y;
+}
+
 bool
 layer_intersects(struct liftoff_layer *a, struct liftoff_layer *b)
 {
@@ -179,8 +202,7 @@ layer_intersects(struct liftoff_layer *a, struct liftoff_layer *b)
 	layer_get_rect(a, &ra);
 	layer_get_rect(b, &rb);
 
-	return ra.x < rb.x + rb.width && ra.y < rb.y + rb.height &&
+	return rect_intersects(&ra, &rb);
-	       ra.x + ra.width > rb.x && ra.y + ra.height > rb.y;
 }
 
 void