xdg-positioner: honor constraint flags

rootston/xdg_shell_v6.c

@@ -51,6 +51,9 @@ static void popup_handle_new_popup(struct wl_listener *listener, void *data) {
 }
 
 static void popup_unconstrain(struct roots_xdg_popup_v6 *popup) {
+	// get the output of the popup's positioner anchor point and convert it to
+	// the toplevel parent's coordinate system and then pass it to
+	// wlr_xdg_popup_v6_unconstrain_from_box
 	struct roots_view *view = popup->view_child.view;
 	struct wlr_output_layout *layout = view->desktop->layout;
 	struct wlr_xdg_popup_v6 *wlr_popup = popup->wlr_popup;

types/wlr_xdg_shell_v6.c

@@ -1723,28 +1723,34 @@ static bool wlr_xdg_popup_v6_unconstrain_flip(struct wlr_xdg_popup_v6 *popup,
 		return true;
 	}
 
-	if (offset_x) {
+	bool flip_x = offset_x &&
+		(popup->positioner.constraint_adjustment &
+		 WLR_POSITIONER_V6_CONSTRAINT_ADJUSTMENT_FLIP_X);
+
+	bool flip_y = offset_x &&
+		(popup->positioner.constraint_adjustment &
+		 WLR_POSITIONER_V6_CONSTRAINT_ADJUSTMENT_FLIP_Y);
+
+	if (flip_x) {
 		wlr_positioner_v6_invert_x(&popup->positioner);
 	}
-	if (offset_y) {
+	if (flip_y) {
 		wlr_positioner_v6_invert_y(&popup->positioner);
 	}
 
 	popup->geometry =
 		wlr_xdg_positioner_v6_get_geometry(&popup->positioner);
 
-	wlr_xdg_popup_v6_box_constraints(popup, toplevel_box, &offset_x, &offset_y);
-
 	if (!offset_x && !offset_y) {
 		// no longer constrained
 		return true;
 	}
 
 	// revert the positioner back if it didn't fix it and go to the next part
-	if (offset_x) {
+	if (flip_x) {
 		wlr_positioner_v6_invert_x(&popup->positioner);
 	}
-	if (offset_y) {
+	if (flip_y) {
 		wlr_positioner_v6_invert_y(&popup->positioner);
 	}
 
@@ -1763,11 +1769,19 @@ static bool wlr_xdg_popup_v6_unconstrain_slide(struct wlr_xdg_popup_v6 *popup,
 		return true;
 	}
 
-	if (offset_x) {
+	bool slide_x = offset_x &&
+		(popup->positioner.constraint_adjustment &
+		 WLR_POSITIONER_V6_CONSTRAINT_ADJUSTMENT_SLIDE_X);
+
+	bool slide_y = offset_x &&
+		(popup->positioner.constraint_adjustment &
+		 WLR_POSITIONER_V6_CONSTRAINT_ADJUSTMENT_SLIDE_Y);
+
+	if (slide_x) {
 		popup->geometry.x += offset_x;
 	}
 
-	if (offset_y) {
+	if (slide_y) {
 		popup->geometry.y += offset_y;
 	}
 
@@ -1785,10 +1799,18 @@ static bool wlr_xdg_popup_v6_unconstrain_resize(struct wlr_xdg_popup_v6 *popup,
 		return true;
 	}
 
-	if (offset_x) {
+	bool resize_x = offset_x &&
+		(popup->positioner.constraint_adjustment &
+		 WLR_POSITIONER_V6_CONSTRAINT_ADJUSTMENT_RESIZE_X);
+
+	bool resize_y = offset_x &&
+		(popup->positioner.constraint_adjustment &
+		 WLR_POSITIONER_V6_CONSTRAINT_ADJUSTMENT_RESIZE_Y);
+
+	if (resize_x) {
 		popup->geometry.width -= offset_x;
 	}
-	if (offset_y) {
+	if (resize_y) {
 		popup->geometry.height -= offset_y;
 	}
 
@@ -1799,8 +1821,15 @@ static bool wlr_xdg_popup_v6_unconstrain_resize(struct wlr_xdg_popup_v6 *popup,
 
 void wlr_xdg_popup_v6_unconstrain_from_box(struct wlr_xdg_popup_v6 *popup,
 		struct wlr_box *toplevel_box) {
-	wlr_xdg_popup_v6_unconstrain_flip(popup, toplevel_box);
+	if (wlr_xdg_popup_v6_unconstrain_flip(popup, toplevel_box)) {
-	wlr_xdg_popup_v6_unconstrain_slide(popup, toplevel_box);
+		return;
+	}
+	if (wlr_xdg_popup_v6_unconstrain_slide(popup, toplevel_box)) {
+		return;
+	}
+	if (wlr_xdg_popup_v6_unconstrain_resize(popup, toplevel_box)) {
+		return;
+	}
 }
 
 void wlr_positioner_v6_invert_x(struct wlr_xdg_positioner_v6_attributes *positioner) {