Path: blob/main/gpu_display/src/gpu_display_win/mouse_input_manager.rs
5394 views
// Copyright 2023 The ChromiumOS Authors1// Use of this source code is governed by a BSD-style license that can be2// found in the LICENSE file.34use std::ffi::c_void;5use std::mem;6use std::ops::ControlFlow;7use std::ptr::null;8use std::ptr::null_mut;910use anyhow::Context;11use anyhow::Result;12use base::error;13use base::info;14use base::warn;15use euclid::point2;16use euclid::size2;17use euclid::Box2D;18use euclid::Point2D;19use euclid::Size2D;20use euclid::Transform2D;21use euclid::Vector2D;22use linux_input_sys::virtio_input_event;23use smallvec::smallvec;24use smallvec::SmallVec;25use win_util::syscall_bail;26use winapi::shared::minwindef::LOWORD;27use winapi::shared::minwindef::LPARAM;28use winapi::shared::minwindef::LRESULT;29use winapi::shared::minwindef::UINT;30use winapi::shared::windef::RECT;31use winapi::shared::windowsx::GET_X_LPARAM;32use winapi::shared::windowsx::GET_Y_LPARAM;33use winapi::um::errhandlingapi::GetLastError;34use winapi::um::winuser::ClipCursor;35use winapi::um::winuser::GetRawInputData;36use winapi::um::winuser::GetSystemMetrics;37use winapi::um::winuser::IntersectRect;38use winapi::um::winuser::ReleaseCapture;39use winapi::um::winuser::SetCapture;40use winapi::um::winuser::SetCursor;41use winapi::um::winuser::SetRect;42use winapi::um::winuser::GET_WHEEL_DELTA_WPARAM;43use winapi::um::winuser::HRAWINPUT;44use winapi::um::winuser::HTCLIENT;45use winapi::um::winuser::MA_ACTIVATE;46use winapi::um::winuser::MA_NOACTIVATE;47use winapi::um::winuser::MK_LBUTTON;48use winapi::um::winuser::MOUSE_MOVE_ABSOLUTE;49use winapi::um::winuser::MOUSE_MOVE_RELATIVE;50use winapi::um::winuser::MOUSE_VIRTUAL_DESKTOP;51use winapi::um::winuser::RAWINPUT;52use winapi::um::winuser::RAWINPUTHEADER;53use winapi::um::winuser::RID_INPUT;54use winapi::um::winuser::RIM_TYPEMOUSE;55use winapi::um::winuser::SM_CXSCREEN;56use winapi::um::winuser::SM_CXVIRTUALSCREEN;57use winapi::um::winuser::SM_CYSCREEN;58use winapi::um::winuser::SM_CYVIRTUALSCREEN;59use winapi::um::winuser::SM_XVIRTUALSCREEN;60use winapi::um::winuser::SM_YVIRTUALSCREEN;61use winapi::um::winuser::WHEEL_DELTA;6263use super::math_util::Rect;64use super::math_util::RectExtension;65use super::window::BasicWindow;66use super::window::GuiWindow;67use super::window_message_dispatcher::DisplayEventDispatcher;68use super::window_message_processor::MouseMessage;69use super::window_message_processor::WindowMessage;70use super::window_message_processor::WindowPosMessage;71use super::HostWindowSpace;72use super::MouseMode;73use super::VirtualDisplaySpace;74use crate::EventDeviceKind;7576// Used as the multi-touch slot & tracking IDs.77// See https://www.kernel.org/doc/html/latest/input/multi-touch-protocol.html for further details.78const PRIMARY_FINGER_ID: i32 = 0;7980// The fixed amount of pixels to remove in each side of the client window when confining the cursor81// in relative mouse mode82const BORDER_OFFSET: i32 = 10;8384/// Responsible for capturing input from a HWND and forwarding it to the guest.85pub(crate) struct MouseInputManager {86display_event_dispatcher: DisplayEventDispatcher,87mouse_pos: Option<Point2D<f64, HostWindowSpace>>,88/// Accumulates the delta value for mouse/touchpad scrolling. The doc for `WHEEL_DELTA` says it89/// "... is the threshold for action to be taken, and one such action (for example, scrolling90/// one increment) should occur for each delta". While the mouse wheel produces exactly91/// `WHEEL_DELTA` every time it is scrolled, a touchpad may produce much smaller amounts, so we92/// would want to accumulate it until `WHEEL_DELTA` is reached.93accumulated_wheel_delta: i16,94mouse_mode: MouseMode,95/// Used to transform coordinates from the host window space to the virtual device space96transform: Transform2D<f64, HostWindowSpace, VirtualDisplaySpace>,97/// A 2D box in virtual device coordinate space. If a touch event happens outside the box, the98/// event won't be processed.99virtual_display_box: Box2D<f64, VirtualDisplaySpace>,100}101102impl MouseInputManager {103pub fn new(104_window: &GuiWindow,105transform: Transform2D<f64, HostWindowSpace, VirtualDisplaySpace>,106virtual_display_size: Size2D<u32, VirtualDisplaySpace>,107display_event_dispatcher: DisplayEventDispatcher,108) -> Self {109let virtual_display_box = Box2D::new(110Point2D::zero(),111Point2D::zero().add_size(&virtual_display_size),112)113.to_f64();114Self {115display_event_dispatcher,116mouse_pos: None,117accumulated_wheel_delta: 0,118mouse_mode: MouseMode::Touchscreen,119transform,120virtual_display_box,121}122}123124/// Processes raw input events for the mouse.125///126/// Raw input is required to properly create relative motion events. A previous version used127/// simulated relative motion based on WM_MOUSEMOVE events (which provide absolute position).128/// That version worked surprisingly well, except it had one fatal flaw: the guest & host129/// pointer are not necessarily in sync. This means the host's pointer can hit the edge of the130/// VM's display window, but the guest's pointer is still in the middle of the screen; for131/// example, the host pointer hits the left edge and stops generating position change events,132/// but the guest pointer is still in the middle of the screen. Because of that desync, the left133/// half of the guest's screen is inaccessible. To avoid that flaw, we use raw input to get134/// the actual relative input events directly from Windows.135#[inline]136pub fn handle_raw_input_event(&mut self, window: &GuiWindow, input_lparam: HRAWINPUT) {137if !self.should_capture_cursor(window) {138return;139}140141let mut promised_size: UINT = 0;142// SAFETY:143// Safe because promised_size is guaranteed to exist.144let ret = unsafe {145GetRawInputData(146input_lparam,147RID_INPUT,148null_mut(),149&mut promised_size as *mut UINT,150mem::size_of::<RAWINPUTHEADER>() as u32,151)152};153if ret == UINT::MAX {154error!(155"Relative mouse error: GetRawInputData failed to get size of events: {}",156// SAFETY: trivially safe157unsafe { GetLastError() }158);159return;160}161if promised_size == 0 {162// No actual raw input to process163return;164}165166// buf should be 8 byte aligned because it is used as a RAWINPUT struct. Note that this167// buffer could be slightly larger, but that's necessary for safety.168let mut buf: Vec<u64> = Vec::with_capacity(169promised_size as usize / mem::size_of::<u64>()170+ promised_size as usize % mem::size_of::<u64>(),171);172let mut buf_size: UINT = promised_size as UINT;173174// SAFETY:175// Safe because buf is guaranteed to exist, and be of sufficient size because we checked the176// required size previously.177let input_size = unsafe {178GetRawInputData(179input_lparam,180RID_INPUT,181buf.as_mut_ptr() as *mut c_void,182&mut buf_size as *mut UINT,183mem::size_of::<RAWINPUTHEADER>() as u32,184)185};186if input_size == UINT::MAX {187error!(188"Relative mouse error: GetRawInputData failed to get events: {}",189// SAFETY: trivially safe190unsafe { GetLastError() }191);192return;193}194if input_size != promised_size {195error!(196"GetRawInputData returned {}, but was expected to return {}.",197input_size, promised_size198);199return;200}201202// SAFETY:203// Safe because buf is guaranteed to exist, and it was correctly populated by the previous204// call to GetRawInputData.205let raw_input = unsafe { (buf.as_ptr() as *const RAWINPUT).as_ref().unwrap() };206207self.process_valid_raw_input_mouse(window, raw_input)208}209210/// Processes a RAWINPUT event for a mouse and dispatches the appropriate virtio_input_events211/// to the guest.212#[inline]213fn process_valid_raw_input_mouse(&mut self, window: &GuiWindow, raw_input: &RAWINPUT) {214if raw_input.header.dwType != RIM_TYPEMOUSE {215error!("Receiving non-mouse RAWINPUT.");216return;217}218219// SAFETY:220// Safe because we checked that raw_input.data is a mouse event.221let mouse = unsafe { raw_input.data.mouse() };222223// MOUSE_MOVE_RELATIVE is a bitwise flag value that is zero; in other words, it is224// considered "set" if the 0th bit is zero. For that reason, we mask off the relevant225// bit, and assert it is equal to the flag value (which is zero).226let mouse_motion = if mouse.usFlags & 0x1 == MOUSE_MOVE_RELATIVE {227// Most mice will report as relative, which makes this simple.228Some(Vector2D::<f64, HostWindowSpace>::new(229mouse.lLastX as f64,230mouse.lLastY as f64,231))232} else if mouse.usFlags & MOUSE_MOVE_ABSOLUTE == MOUSE_MOVE_ABSOLUTE {233// Trackpads may present as "absolute" devices, but we want to show a relative234// device to the guest. We simulate relative motion in that case by figuring out235// how much the mouse has moved relative to its last known position.236// `lLastX` and `lLastY` contain normalized absolute coordinates, and they should be237// mapped to the primary monitor coordinate space first:238// https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-rawmouse#remarks239let primary_monitor_rect = get_primary_monitor_rect(240/* is_virtual_desktop= */241mouse.usFlags & MOUSE_VIRTUAL_DESKTOP == MOUSE_VIRTUAL_DESKTOP,242)243.to_f64();244let new_mouse_pos = point2(245mouse.lLastX as f64 * primary_monitor_rect.width() / 65535.0246+ primary_monitor_rect.min_x(),247mouse.lLastY as f64 * primary_monitor_rect.height() / 65535.0248+ primary_monitor_rect.min_y(),249);250let motion = self.mouse_pos.as_ref().map(|pos| new_mouse_pos - *pos);251self.mouse_pos = Some(new_mouse_pos);252motion253} else {254// Other non-motion events we don't care about.255None256};257258if let Some(mouse_motion) = mouse_motion {259let events = self.create_relative_mouse_events(mouse_motion);260self.display_event_dispatcher.dispatch(261window,262events.as_slice(),263EventDeviceKind::Mouse,264);265}266}267268#[inline]269fn create_relative_mouse_events(270&self,271mouse_motion: Vector2D<f64, HostWindowSpace>,272) -> SmallVec<[virtio_input_event; 2]> {273smallvec![274virtio_input_event::relative_x(mouse_motion.x as i32),275virtio_input_event::relative_y(mouse_motion.y as i32),276]277}278279/// Converts the given host point to a guest point, clipping it to the host window viewport.280#[inline]281fn to_guest_point(282&self,283pos: Point2D<i32, HostWindowSpace>,284) -> Option<Point2D<i32, VirtualDisplaySpace>> {285let pos = self.transform.transform_point(pos.to_f64());286let pos = pos.clamp(self.virtual_display_box.min, self.virtual_display_box.max);287Some(pos.round().to_i32())288}289290/// Takes a down or up event and converts it into suitable multi touch events. Those events are291/// then dispatched to the guest. Note that a "click" and movement of the cursor with a button292/// down are represented as the same event.293fn handle_multi_touch_finger(294&mut self,295window: &GuiWindow,296pos: Point2D<i32, HostWindowSpace>,297pressed: bool,298finger_id: i32,299) {300let pos = match self.to_guest_point(pos) {301Some(pos) => pos,302None => return,303};304if pressed {305self.display_event_dispatcher.dispatch(306window,307&[308virtio_input_event::multitouch_slot(finger_id),309virtio_input_event::multitouch_tracking_id(finger_id),310virtio_input_event::multitouch_absolute_x(pos.x),311virtio_input_event::multitouch_absolute_y(pos.y),312virtio_input_event::touch(pressed),313],314EventDeviceKind::Touchscreen,315);316} else {317self.display_event_dispatcher.dispatch(318window,319&[320virtio_input_event::multitouch_slot(finger_id),321virtio_input_event::multitouch_tracking_id(-1),322virtio_input_event::touch(false),323],324EventDeviceKind::Touchscreen,325);326}327}328329/// Handles WM_MOUSEMOVE events. Note that these events are NOT used for the relative mouse330/// (we use raw input instead).331#[inline]332fn handle_mouse_move(333&mut self,334window: &GuiWindow,335pos: Point2D<i32, HostWindowSpace>,336left_down: bool,337) {338if let MouseMode::Touchscreen = self.mouse_mode {339if left_down {340self.handle_multi_touch_finger(window, pos, left_down, PRIMARY_FINGER_ID);341}342}343}344345/// Sets or releases mouse "capture" when a mouse button is pressed or released. This lets us346/// track motion beyond the window bounds, which is useful for drag gestures in the guest.347fn adjust_capture_on_mouse_button(&self, down: bool, window: &GuiWindow) {348if let MouseMode::Touchscreen = self.mouse_mode {349if down {350// SAFETY: safe because window is alive during the call, and we don't care if the351// function fails to capture the mouse because there's nothing we can do about that352// anyway.353unsafe { SetCapture(window.handle()) };354}355}356357if !down {358// SAFETY: safe because no memory is involved.359if unsafe { ReleaseCapture() } == 0 {360// SAFETY: trivially safe361warn!("failed to release capture: {}", unsafe { GetLastError() });362}363}364}365366fn handle_mouse_button_left(367&mut self,368pos: Point2D<i32, HostWindowSpace>,369down: bool,370window: &GuiWindow,371) {372self.adjust_capture_on_mouse_button(down, window);373match self.mouse_mode {374MouseMode::Touchscreen => {375self.handle_multi_touch_finger(window, pos, down, PRIMARY_FINGER_ID);376}377MouseMode::Relative => {378self.display_event_dispatcher.dispatch(379window,380&[virtio_input_event::left_click(down)],381EventDeviceKind::Mouse,382);383}384}385}386387fn handle_mouse_button_right(&mut self, window: &GuiWindow, down: bool) {388if let MouseMode::Relative = self.mouse_mode {389self.display_event_dispatcher.dispatch(390window,391&[virtio_input_event::right_click(down)],392EventDeviceKind::Mouse,393);394}395}396397fn handle_mouse_button_middle(&mut self, window: &GuiWindow, down: bool) {398if let MouseMode::Relative = self.mouse_mode {399self.display_event_dispatcher.dispatch(400window,401&[virtio_input_event::middle_click(down)],402EventDeviceKind::Mouse,403);404}405}406407fn handle_mouse_button_forward(&mut self, window: &GuiWindow, down: bool) {408if let MouseMode::Relative = self.mouse_mode {409self.display_event_dispatcher.dispatch(410window,411&[virtio_input_event::forward_click(down)],412EventDeviceKind::Mouse,413);414}415}416417fn handle_mouse_button_back(&mut self, window: &GuiWindow, down: bool) {418if let MouseMode::Relative = self.mouse_mode {419self.display_event_dispatcher.dispatch(420window,421&[virtio_input_event::back_click(down)],422EventDeviceKind::Mouse,423);424}425}426427fn set_mouse_mode(&mut self, window: &GuiWindow, mode: MouseMode) {428info!(429"requested mouse mode switch to {:?} (current mode is: {:?})",430mode, self.mouse_mode431);432if mode == self.mouse_mode {433return;434}435436self.mouse_mode = mode;437self.mouse_pos = None;438if let Err(e) = self.adjust_cursor_capture(window) {439error!(440"Failed to adjust cursor capture on mouse mode change: {:?}",441e442)443}444}445446fn handle_mouse_wheel(447&mut self,448window: &GuiWindow,449z_delta: i16,450_cursor_pos: Option<Point2D<i32, HostWindowSpace>>,451) {452let accumulated_delta = self.accumulated_wheel_delta + z_delta;453self.accumulated_wheel_delta = accumulated_delta % WHEEL_DELTA;454let scaled_delta = accumulated_delta / WHEEL_DELTA;455if scaled_delta == 0 {456return;457}458let deivce_kind = match self.mouse_mode {459MouseMode::Relative => EventDeviceKind::Mouse,460MouseMode::Touchscreen => return,461};462self.display_event_dispatcher.dispatch(463window,464&[virtio_input_event::wheel(scaled_delta as i32)],465deivce_kind,466);467}468469pub fn update_host_to_guest_transform(470&mut self,471transform: Transform2D<f64, HostWindowSpace, VirtualDisplaySpace>,472) {473self.transform = transform;474}475476/// Possible return values:477/// 1. `ControlFlow::Continue`, should continue invoking other modules, such as the window478/// manager, to perform more processing.479/// 2. `ControlFlow::Break(Some)`, should skip any other processing and return the value.480/// 3. `ControlFlow::Break(None)`, should immediately perform default processing.481#[inline]482pub fn handle_window_message(483&mut self,484window: &GuiWindow,485message: &WindowMessage,486) -> ControlFlow<Option<LRESULT>> {487match message {488WindowMessage::WindowActivate { .. }489| WindowMessage::WindowPos(WindowPosMessage::EnterSizeMove)490| WindowMessage::WindowPos(WindowPosMessage::ExitSizeMove)491| WindowMessage::WindowPos(WindowPosMessage::WindowPosChanged { .. }) => {492if let Err(e) = self.adjust_cursor_capture(window) {493error!("Failed to adjust cursor capture on {:?}: {:?}", message, e)494}495}496WindowMessage::Mouse(mouse_message) => {497return self.handle_mouse_message(window, mouse_message);498}499_ => (),500}501ControlFlow::Continue(())502}503504/// Possible return values are documented at `handle_window_message()`.505#[inline]506fn handle_mouse_message(507&mut self,508window: &GuiWindow,509message: &MouseMessage,510) -> ControlFlow<Option<LRESULT>> {511match message {512MouseMessage::MouseMove { w_param, l_param } => {513// Safe because `l_param` comes from the window message and should contain valid514// numbers.515let (x, y) = get_x_y_from_lparam(*l_param);516self.handle_mouse_move(517window,518Point2D::<_, HostWindowSpace>::new(x, y),519w_param & MK_LBUTTON != 0,520);521}522MouseMessage::LeftMouseButton { is_down, l_param } => {523// Safe because `l_param` comes from the window message and should contain valid524// numbers.525let (x, y) = get_x_y_from_lparam(*l_param);526self.handle_mouse_button_left(527Point2D::<_, HostWindowSpace>::new(x, y),528*is_down,529window,530);531}532MouseMessage::RightMouseButton { is_down } => {533self.handle_mouse_button_right(window, *is_down)534}535MouseMessage::MiddleMouseButton { is_down } => {536self.handle_mouse_button_middle(window, *is_down)537}538MouseMessage::ForwardMouseButton { is_down } => {539self.handle_mouse_button_forward(window, *is_down)540}541MouseMessage::BackMouseButton { is_down } => {542self.handle_mouse_button_back(window, *is_down)543}544MouseMessage::MouseWheel { w_param, l_param } => {545// Safe because `l_param` comes from the window message and should contain valid546// numbers.547let (x, y) = get_x_y_from_lparam(*l_param);548let cursor_pos = window.screen_to_client(Point2D::new(x, y));549if let Err(ref e) = cursor_pos {550error!(551"Failed to convert cursor position to client coordinates: {}",552e553);554}555556let z_delta = GET_WHEEL_DELTA_WPARAM(*w_param);557self.handle_mouse_wheel(window, z_delta, cursor_pos.ok());558}559MouseMessage::SetCursor => {560return if self.should_capture_cursor(window) {561// Hide the cursor and skip default processing.562// SAFETY: trivially safe563unsafe { SetCursor(null_mut()) };564ControlFlow::Continue(())565} else {566// Request default processing, i.e. showing the cursor.567ControlFlow::Break(None)568};569}570MouseMessage::MouseActivate { l_param } => {571let hit_test = LOWORD(*l_param as u32) as isize;572// Only activate if we hit the client area.573let activate = if hit_test == HTCLIENT {574MA_ACTIVATE575} else {576MA_NOACTIVATE577};578return ControlFlow::Break(Some(activate as LRESULT));579}580}581ControlFlow::Continue(())582}583584pub fn handle_change_mouse_mode_request(&mut self, window: &GuiWindow, mouse_mode: MouseMode) {585self.set_mouse_mode(window, mouse_mode);586}587588/// Confines/releases the cursor to/from `window`, depending on the current mouse mode and589/// window states.590fn adjust_cursor_capture(&mut self, window: &GuiWindow) -> Result<()> {591let should_capture = self.should_capture_cursor(window);592if should_capture {593self.confine_cursor_to_window_internal(window)594.context("When confining cursor to window")?;595} else {596// SAFETY: trivially safe597unsafe {598clip_cursor(null()).context("When releasing cursor from window")?;599}600}601Ok(())602}603604/// Confines the host cursor to a new window area.605fn confine_cursor_to_window_internal(&mut self, window: &GuiWindow) -> Result<()> {606let work_rect = window.get_monitor_info()?.work_rect.to_sys_rect();607let client_rect = window.get_client_rect()?;608609// Translate client rect to screen coordinates.610let client_ul = window.client_to_screen(&client_rect.min())?;611let client_br = window.client_to_screen(&client_rect.max())?;612let mut client_rect = client_rect.to_sys_rect();613614// SAFETY:615// Safe because hwnd and all RECT are valid objects616unsafe {617SetRect(618&mut client_rect,619client_ul.x + BORDER_OFFSET,620client_ul.y + BORDER_OFFSET,621client_br.x - BORDER_OFFSET,622client_br.y - BORDER_OFFSET,623);624let mut clip_rect = RECT::default();625626// If client_rect intersects with the taskbar then remove that area.627if IntersectRect(&mut clip_rect, &client_rect, &work_rect) != 0 {628clip_cursor(&clip_rect)?;629} else {630clip_cursor(&client_rect)?;631}632}633Ok(())634}635636/// Returns whether we intend the mouse cursor to be captured based on the mouse mode.637///638/// Note that we also need to check the window state to see if we actually want to capture the639/// cursor at this moment. See `should_capture_cursor()`.640fn is_capture_mode(&self) -> bool {641self.mouse_mode == MouseMode::Relative642}643644/// Returns true if the mouse cursor should be captured and hidden.645///646/// We don't always want mouse capture in relative mouse mode. When we are dragging the title647/// bar to move the window around, or dragging window borders/corners for resizing, we'd still648/// want to show the cursor until the dragging ends.649fn should_capture_cursor(&self, window: &GuiWindow) -> bool {650self.is_capture_mode()651&& window.is_global_foreground_window()652&& !window.is_sizing_or_moving()653}654}655656/// Given an l_param from a mouse event, extracts the (x, y) coordinates. Note that these657/// coordinates should be positive provided the mouse is not captured. (They can be negative when658/// the mouse is captured and it moves outside the bounds of the hwnd.)659fn get_x_y_from_lparam(l_param: LPARAM) -> (i32, i32) {660(GET_X_LPARAM(l_param), GET_Y_LPARAM(l_param))661}662663unsafe fn clip_cursor(rect: *const RECT) -> Result<()> {664if ClipCursor(rect) == 0 {665syscall_bail!("Failed to call ClipCursor()");666}667Ok(())668}669670fn get_primary_monitor_rect(is_virtual_desktop: bool) -> Rect {671// SAFETY: trivially-safe672let (origin, size) = unsafe {673if is_virtual_desktop {674(675point2(676GetSystemMetrics(SM_XVIRTUALSCREEN),677GetSystemMetrics(SM_YVIRTUALSCREEN),678),679size2(680GetSystemMetrics(SM_CXVIRTUALSCREEN),681GetSystemMetrics(SM_CYVIRTUALSCREEN),682),683)684} else {685(686Point2D::zero(),687size2(GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)),688)689}690};691Rect::new(origin, size)692}693694695