wlr_box: properly calculate closest point for non-positive area boxes

If box->width/height is <= 0, the box doesn't contain any points, and so
there is no closest point. wlr_box_closest_point should return NAN in this
case.

In addition, we need to handle empty boxes in a few other
output-layout-related places, because outputs can have size 0x0 when
they are created or destroyed.

types/wlr_box.c

@@ -8,6 +8,13 @@
 
 void wlr_box_closest_point(const struct wlr_box *box, double x, double y,
 		double *dest_x, double *dest_y) {
+	// if box is empty, then it contains no points, so no closest point either
+	if (box->width <= 0 || box->height <= 0) {
+		*dest_x = NAN;
+		*dest_y = NAN;
+		return;
+	}
+
 	// find the closest x point
 	if (x < box->x) {
 		*dest_x = box->x;

types/wlr_cursor.c

@@ -278,6 +278,10 @@ void wlr_cursor_warp_closest(struct wlr_cursor *cur,
 	struct wlr_box *mapping = get_mapping(cur, dev);
 	if (mapping) {
 		wlr_box_closest_point(mapping, lx, ly, &lx, &ly);
+		if (isnan(lx) || isnan(ly)) {
+			lx = 0;
+			ly = 0;
+		}
 	} else {
 		wlr_output_layout_closest_point(cur->state->layout, NULL, lx, ly,
 			&lx, &ly);

types/wlr_output_layout.c

@@ -328,7 +328,7 @@ void wlr_output_layout_closest_point(struct wlr_output_layout *layout,
 		return;
 	}
 
-	double min_x = DBL_MAX, min_y = DBL_MAX, min_distance = DBL_MAX;
+	double min_x = 0, min_y = 0, min_distance = DBL_MAX;
 	struct wlr_output_layout_output *l_output;
 	wl_list_for_each(l_output, &layout->outputs, link) {
 		if (reference != NULL && reference != l_output->output) {
@@ -347,7 +347,7 @@ void wlr_output_layout_closest_point(struct wlr_output_layout *layout,
 			output_distance = DBL_MAX;
 		}
 
-		if (output_distance <= min_distance) {
+		if (output_distance < min_distance) {
 			min_x = output_x;
 			min_y = output_y;
 			min_distance = output_distance;