Add mutitouch support for touch screens

src/core/frontend/emu_window.cpp

@@ -21,21 +21,17 @@ public:
 
     std::mutex mutex;
 
-    bool touch_pressed = false; ///< True if touchpad area is currently pressed, otherwise false
-
-    float touch_x = 0.0f; ///< Touchpad X-position
-    float touch_y = 0.0f; ///< Touchpad Y-position
+    Input::TouchStatus status;
 
 private:
     class Device : public Input::TouchDevice {
     public:
         explicit Device(std::weak_ptr<TouchState>&& touch_state) : touch_state(touch_state) {}
         Input::TouchStatus GetStatus() const override {
-            Input::TouchStatus touch_status = {};
+            Input::TouchStatus touch_status{};
             if (auto state = touch_state.lock()) {
                 std::lock_guard guard{state->mutex};
-                touch_status[0] =
-                    std::make_tuple(state->touch_x, state->touch_y, state->touch_pressed);
+                touch_status = state->status;
             }
             return touch_status;
         }
@@ -81,36 +77,44 @@ std::tuple<unsigned, unsigned> EmuWindow::ClipToTouchScreen(unsigned new_x, unsi
     return std::make_tuple(new_x, new_y);
 }
 
-void EmuWindow::TouchPressed(unsigned framebuffer_x, unsigned framebuffer_y) {
-    if (!IsWithinTouchscreen(framebuffer_layout, framebuffer_x, framebuffer_y))
+void EmuWindow::TouchPressed(unsigned framebuffer_x, unsigned framebuffer_y, std::size_t id) {
+    if (!IsWithinTouchscreen(framebuffer_layout, framebuffer_x, framebuffer_y)) {
         return;
+    }
+    if (id >= touch_state->status.size()) {
+        return;
+    }
 
     std::lock_guard guard{touch_state->mutex};
-    touch_state->touch_x =
+    const float x =
         static_cast<float>(framebuffer_x - framebuffer_layout.screen.left) /
         static_cast<float>(framebuffer_layout.screen.right - framebuffer_layout.screen.left);
-    touch_state->touch_y =
+    const float y =
         static_cast<float>(framebuffer_y - framebuffer_layout.screen.top) /
         static_cast<float>(framebuffer_layout.screen.bottom - framebuffer_layout.screen.top);
 
-    touch_state->touch_pressed = true;
+    touch_state->status[id] = std::make_tuple(x, y, true);
 }
 
-void EmuWindow::TouchReleased() {
+void EmuWindow::TouchReleased(std::size_t id) {
+    if (id >= touch_state->status.size()) {
+        return;
+    }
     std::lock_guard guard{touch_state->mutex};
-    touch_state->touch_pressed = false;
-    touch_state->touch_x = 0;
-    touch_state->touch_y = 0;
+    touch_state->status[id] = std::make_tuple(0.0f, 0.0f, false);
 }
 
-void EmuWindow::TouchMoved(unsigned framebuffer_x, unsigned framebuffer_y) {
-    if (!touch_state->touch_pressed)
+void EmuWindow::TouchMoved(unsigned framebuffer_x, unsigned framebuffer_y, std::size_t id) {
+    if (id >= touch_state->status.size()) {
+        return;
+    }
+    if (!std::get<2>(touch_state->status[id]))
         return;
 
     if (!IsWithinTouchscreen(framebuffer_layout, framebuffer_x, framebuffer_y))
         std::tie(framebuffer_x, framebuffer_y) = ClipToTouchScreen(framebuffer_x, framebuffer_y);
 
-    TouchPressed(framebuffer_x, framebuffer_y);
+    TouchPressed(framebuffer_x, framebuffer_y, id);
 }
 
 void EmuWindow::UpdateCurrentFramebufferLayout(unsigned width, unsigned height) {

src/core/frontend/emu_window.h

@@ -117,18 +117,22 @@ public:
      * Signal that a touch pressed event has occurred (e.g. mouse click pressed)
      * @param framebuffer_x Framebuffer x-coordinate that was pressed
      * @param framebuffer_y Framebuffer y-coordinate that was pressed
+     * @param id Touch event id
      */
-    void TouchPressed(unsigned framebuffer_x, unsigned framebuffer_y);
+    void TouchPressed(unsigned framebuffer_x, unsigned framebuffer_y, std::size_t id);
 
-    /// Signal that a touch released event has occurred (e.g. mouse click released)
-    void TouchReleased();
+    /** Signal that a touch released event has occurred (e.g. mouse click released)
+     *@param id Touch event id
+     */
+    void TouchReleased(std::size_t id);
 
     /**
      * Signal that a touch movement event has occurred (e.g. mouse was moved over the emu window)
      * @param framebuffer_x Framebuffer x-coordinate
      * @param framebuffer_y Framebuffer y-coordinate
+     * @param id Touch event id
      */
-    void TouchMoved(unsigned framebuffer_x, unsigned framebuffer_y);
+    void TouchMoved(unsigned framebuffer_x, unsigned framebuffer_y, std::size_t id);
 
     /**
      * Returns currently active configuration.

src/core/hle/service/hid/controllers/touchscreen.cpp

@@ -18,7 +18,7 @@ Controller_Touchscreen::Controller_Touchscreen(Core::System& system) : Controlle
 Controller_Touchscreen::~Controller_Touchscreen() = default;
 
 void Controller_Touchscreen::OnInit() {
-    for (size_t id = 0; id < MAX_FINGERS; id++) {
+    for (std::size_t id = 0; id < MAX_FINGERS; ++id) {
         mouse_finger_id[id] = MAX_FINGERS;
         keyboard_finger_id[id] = MAX_FINGERS;
         udp_finger_id[id] = MAX_FINGERS;
@@ -48,23 +48,29 @@ void Controller_Touchscreen::OnUpdate(const Core::Timing::CoreTiming& core_timin
     cur_entry.sampling_number2 = cur_entry.sampling_number;
 
     const Input::TouchStatus& mouse_status = touch_mouse_device->GetStatus();
-    const Input::TouchStatus& keyboard_status = touch_btn_device->GetStatus();
     const Input::TouchStatus& udp_status = touch_udp_device->GetStatus();
-    for (size_t id = 0; id < mouse_status.size(); id++) {
+    for (std::size_t id = 0; id < mouse_status.size(); ++id) {
         mouse_finger_id[id] = UpdateTouchInputEvent(mouse_status[id], mouse_finger_id[id]);
-        keyboard_finger_id[id] = UpdateTouchInputEvent(keyboard_status[id], keyboard_finger_id[id]);
         udp_finger_id[id] = UpdateTouchInputEvent(udp_status[id], udp_finger_id[id]);
     }
 
+    if (Settings::values.use_touch_from_button) {
+        const Input::TouchStatus& keyboard_status = touch_btn_device->GetStatus();
+        for (std::size_t id = 0; id < mouse_status.size(); ++id) {
+            keyboard_finger_id[id] =
+                UpdateTouchInputEvent(keyboard_status[id], keyboard_finger_id[id]);
+        }
+    }
+
     std::array<Finger, 16> active_fingers;
     const auto end_iter = std::copy_if(fingers.begin(), fingers.end(), active_fingers.begin(),
                                        [](const auto& finger) { return finger.pressed; });
     const auto active_fingers_count =
-        static_cast<size_t>(std::distance(active_fingers.begin(), end_iter));
+        static_cast<std::size_t>(std::distance(active_fingers.begin(), end_iter));
 
     const u64 tick = core_timing.GetCPUTicks();
     cur_entry.entry_count = static_cast<s32_le>(active_fingers_count);
-    for (size_t id = 0; id < MAX_FINGERS; id++) {
+    for (std::size_t id = 0; id < MAX_FINGERS; ++id) {
         auto& touch_entry = cur_entry.states[id];
         if (id < active_fingers_count) {
             touch_entry.x = static_cast<u16>(active_fingers[id].x * Layout::ScreenUndocked::Width);
@@ -73,7 +79,7 @@ void Controller_Touchscreen::OnUpdate(const Core::Timing::CoreTiming& core_timin
             touch_entry.diameter_y = Settings::values.touchscreen.diameter_y;
             touch_entry.rotation_angle = Settings::values.touchscreen.rotation_angle;
             touch_entry.delta_time = tick - active_fingers[id].last_touch;
-            active_fingers[id].last_touch = tick;
+            fingers[active_fingers[id].id].last_touch = tick;
             touch_entry.finger = active_fingers[id].id;
             touch_entry.attribute.raw = active_fingers[id].attribute.raw;
         } else {
@@ -101,8 +107,8 @@ void Controller_Touchscreen::OnLoadInputDevices() {
     }
 }
 
-std::optional<size_t> Controller_Touchscreen::GetUnusedFingerID() const {
-    size_t first_free_id = 0;
+std::optional<std::size_t> Controller_Touchscreen::GetUnusedFingerID() const {
+    std::size_t first_free_id = 0;
     while (first_free_id < MAX_FINGERS) {
         if (!fingers[first_free_id].pressed) {
             return first_free_id;
@@ -113,11 +119,11 @@ std::optional<size_t> Controller_Touchscreen::GetUnusedFingerID() const {
     return std::nullopt;
 }
 
-size_t Controller_Touchscreen::UpdateTouchInputEvent(
-    const std::tuple<float, float, bool>& touch_input, size_t finger_id) {
+std::size_t Controller_Touchscreen::UpdateTouchInputEvent(
+    const std::tuple<float, float, bool>& touch_input, std::size_t finger_id) {
     const auto& [x, y, pressed] = touch_input;
     if (pressed) {
-        Attributes attribute = {};
+        Attributes attribute{};
         if (finger_id == MAX_FINGERS) {
             const auto first_free_id = GetUnusedFingerID();
             if (!first_free_id) {

src/core/hle/service/hid/controllers/touchscreen.h

@@ -30,17 +30,17 @@ public:
     void OnLoadInputDevices() override;
 
 private:
-    static constexpr size_t MAX_FINGERS = 16;
+    static constexpr std::size_t MAX_FINGERS = 16;
 
     // Returns an unused finger id, if there is no fingers available std::nullopt will be returned
-    std::optional<size_t> GetUnusedFingerID() const;
+    std::optional<std::size_t> GetUnusedFingerID() const;
 
     // If the touch is new it tries to assing a new finger id, if there is no fingers avaliable no
     // changes will be made. Updates the coordinates if the finger id it's already set. If the touch
     // ends delays the output by one frame to set the end_touch flag before finally freeing the
     // finger id
-    size_t UpdateTouchInputEvent(const std::tuple<float, float, bool>& touch_input,
-                                 size_t finger_id);
+    std::size_t UpdateTouchInputEvent(const std::tuple<float, float, bool>& touch_input,
+                                      std::size_t finger_id);
 
     struct Attributes {
         union {
@@ -92,9 +92,9 @@ private:
     std::unique_ptr<Input::TouchDevice> touch_mouse_device;
     std::unique_ptr<Input::TouchDevice> touch_udp_device;
     std::unique_ptr<Input::TouchDevice> touch_btn_device;
-    std::array<size_t, MAX_FINGERS> mouse_finger_id;
-    std::array<size_t, MAX_FINGERS> keyboard_finger_id;
-    std::array<size_t, MAX_FINGERS> udp_finger_id;
+    std::array<std::size_t, MAX_FINGERS> mouse_finger_id;
+    std::array<std::size_t, MAX_FINGERS> keyboard_finger_id;
+    std::array<std::size_t, MAX_FINGERS> udp_finger_id;
     std::array<Finger, MAX_FINGERS> fingers;
 };
 } // namespace Service::HID

src/input_common/touch_from_button.cpp

@@ -26,8 +26,8 @@ public:
     }
 
     Input::TouchStatus GetStatus() const override {
-        Input::TouchStatus touch_status = {};
-        for (size_t id = 0; id < map.size() && id < touch_status.size(); id++) {
+        Input::TouchStatus touch_status{};
+        for (std::size_t id = 0; id < map.size() && id < touch_status.size(); ++id) {
             const bool state = std::get<0>(map[id])->GetStatus();
             if (state) {
                 const float x = static_cast<float>(std::get<1>(map[id])) /

src/input_common/udp/client.cpp

@@ -136,8 +136,8 @@ static void SocketLoop(Socket* socket) {
 
 Client::Client() {
     LOG_INFO(Input, "Udp Initialization started");
-    for (size_t id = 0; id < MAX_TOUCH_FINGERS; id++) {
-        finger_id[id] = MAX_UDP_CLIENTS * 2;
+    for (std::size_t id = 0; id < MAX_TOUCH_FINGERS; ++id) {
+        finger_id[id] = MAX_TOUCH_FINGERS;
     }
     ReloadSockets();
 }
@@ -262,7 +262,7 @@ void Client::OnPadData(Response::PadData data, std::size_t client) {
         std::lock_guard guard(clients[client].status.update_mutex);
         clients[client].status.motion_status = clients[client].motion.GetMotion();
 
-        for (size_t id = 0; id < data.touch.size(); id++) {
+        for (std::size_t id = 0; id < data.touch.size(); ++id) {
             UpdateTouchInput(data.touch[id], client, id);
         }
 
@@ -314,7 +314,7 @@ void Client::UpdateYuzuSettings(std::size_t client, const Common::Vec3<float>& a
         .port = clients[client].port,
         .pad_index = clients[client].pad_index,
     };
-    for (size_t i = 0; i < 3; ++i) {
+    for (std::size_t i = 0; i < 3; ++i) {
         if (gyro[i] > 5.0f || gyro[i] < -5.0f) {
             pad.motion = static_cast<PadMotion>(i);
             pad.motion_value = gyro[i];
@@ -328,8 +328,8 @@ void Client::UpdateYuzuSettings(std::size_t client, const Common::Vec3<float>& a
     }
 }
 
-std::optional<size_t> Client::GetUnusedFingerID() const {
-    size_t first_free_id = 0;
+std::optional<std::size_t> Client::GetUnusedFingerID() const {
+    std::size_t first_free_id = 0;
     while (first_free_id < MAX_TOUCH_FINGERS) {
         if (!std::get<2>(touch_status[first_free_id])) {
             return first_free_id;
@@ -340,7 +340,7 @@ std::optional<size_t> Client::GetUnusedFingerID() const {
     return std::nullopt;
 }
 
-void Client::UpdateTouchInput(Response::TouchPad& touch_pad, size_t client, size_t id) {
+void Client::UpdateTouchInput(Response::TouchPad& touch_pad, std::size_t client, std::size_t id) {
     // TODO: Use custom calibration per device
     const Common::ParamPackage touch_param(Settings::values.touch_device);
     const u16 min_x = static_cast<u16>(touch_param.Get("min_x", 100));
@@ -367,10 +367,7 @@ void Client::UpdateTouchInput(Response::TouchPad& touch_pad, size_t client, size
     }
 
     if (finger_id[client * 2 + id] != MAX_TOUCH_FINGERS) {
-        auto& [x, y, pressed] = touch_status[finger_id[client * 2 + id]];
-        x = 0;
-        y = 0;
-        pressed = false;
+        touch_status[finger_id[client * 2 + id]] = {};
         finger_id[client * 2 + id] = MAX_TOUCH_FINGERS;
     }
 }

src/input_common/udp/client.h

@@ -28,8 +28,8 @@ class Socket;
 namespace Response {
 struct PadData;
 struct PortInfo;
-struct Version;
 struct TouchPad;
+struct Version;
 } // namespace Response
 
 enum class PadMotion {
@@ -129,10 +129,10 @@ private:
 
     // Returns an unused finger id, if there is no fingers available std::nullopt will be
     // returned
-    std::optional<size_t> GetUnusedFingerID() const;
+    std::optional<std::size_t> GetUnusedFingerID() const;
 
     // Merges and updates all touch inputs into the touch_status array
-    void UpdateTouchInput(Response::TouchPad& touch_pad, size_t client, size_t id);
+    void UpdateTouchInput(Response::TouchPad& touch_pad, std::size_t client, std::size_t id);
 
     bool configuring = false;
 
@@ -143,7 +143,7 @@ private:
     std::array<ClientData, MAX_UDP_CLIENTS> clients{};
     Common::SPSCQueue<UDPPadStatus> pad_queue{};
     Input::TouchStatus touch_status{};
-    std::array<size_t, MAX_TOUCH_FINGERS> finger_id{};
+    std::array<std::size_t, MAX_TOUCH_FINGERS> finger_id{};
 };
 
 /// An async job allowing configuration of the touchpad calibration.

src/yuzu/bootmanager.cpp

@@ -394,7 +394,7 @@ void GRenderWindow::mousePressEvent(QMouseEvent* event) {
     input_subsystem->GetMouse()->PressButton(x, y, event->button());
 
     if (event->button() == Qt::LeftButton) {
-        this->TouchPressed(x, y);
+        this->TouchPressed(x, y, 0);
     }
 
     emit MouseActivity();
@@ -409,7 +409,7 @@ void GRenderWindow::mouseMoveEvent(QMouseEvent* event) {
     auto pos = event->pos();
     const auto [x, y] = ScaleTouch(pos);
     input_subsystem->GetMouse()->MouseMove(x, y);
-    this->TouchMoved(x, y);
+    this->TouchMoved(x, y, 0);
 
     emit MouseActivity();
 }
@@ -423,36 +423,71 @@ void GRenderWindow::mouseReleaseEvent(QMouseEvent* event) {
     input_subsystem->GetMouse()->ReleaseButton(event->button());
 
     if (event->button() == Qt::LeftButton) {
-        this->TouchReleased();
+        this->TouchReleased(0);
     }
 }
 
 void GRenderWindow::TouchBeginEvent(const QTouchEvent* event) {
-    // TouchBegin always has exactly one touch point, so take the .first()
-    const auto [x, y] = ScaleTouch(event->touchPoints().first().pos());
-    this->TouchPressed(x, y);
+    QList<QTouchEvent::TouchPoint> touch_points = event->touchPoints();
+    for (const auto& touch_point : touch_points) {
+        if (!TouchUpdate(touch_point)) {
+            TouchStart(touch_point);
+        }
+    }
 }
 
 void GRenderWindow::TouchUpdateEvent(const QTouchEvent* event) {
-    QPointF pos;
-    int active_points = 0;
-
-    // average all active touch points
-    for (const auto& tp : event->touchPoints()) {
-        if (tp.state() & (Qt::TouchPointPressed | Qt::TouchPointMoved | Qt::TouchPointStationary)) {
-            active_points++;
-            pos += tp.pos();
+    QList<QTouchEvent::TouchPoint> touch_points = event->touchPoints();
+    for (const auto& touch_point : touch_points) {
+        if (!TouchUpdate(touch_point)) {
+            TouchStart(touch_point);
+        }
+    }
+    // Release all inactive points
+    for (std::size_t id = 0; id < touch_ids.size(); ++id) {
+        if (!TouchExist(touch_ids[id], touch_points)) {
+            touch_ids[id] = 0;
+            this->TouchReleased(id + 1);
         }
     }
-
-    pos /= active_points;
-
-    const auto [x, y] = ScaleTouch(pos);
-    this->TouchMoved(x, y);
 }
 
 void GRenderWindow::TouchEndEvent() {
-    this->TouchReleased();
+    for (std::size_t id = 0; id < touch_ids.size(); ++id) {
+        if (touch_ids[id] != 0) {
+            touch_ids[id] = 0;
+            this->TouchReleased(id + 1);
+        }
+    }
+}
+
+bool GRenderWindow::TouchStart(const QTouchEvent::TouchPoint& touch_point) {
+    for (std::size_t id = 0; id < touch_ids.size(); ++id) {
+        if (touch_ids[id] == 0) {
+            touch_ids[id] = touch_point.id() + 1;
+            const auto [x, y] = ScaleTouch(touch_point.pos());
+            this->TouchPressed(x, y, id + 1);
+            return true;
+        }
+    }
+    return false;
+}
+
+bool GRenderWindow::TouchUpdate(const QTouchEvent::TouchPoint& touch_point) {
+    for (std::size_t id = 0; id < touch_ids.size(); ++id) {
+        if (touch_ids[id] == touch_point.id() + 1) {
+            const auto [x, y] = ScaleTouch(touch_point.pos());
+            this->TouchMoved(x, y, id + 1);
+            return true;
+        }
+    }
+    return false;
+}
+
+bool GRenderWindow::TouchExist(std::size_t id,
+                               const QList<QTouchEvent::TouchPoint>& touch_points) const {
+    return std::any_of(touch_points.begin(), touch_points.end(),
+                       [id](const auto& point) { return id == point.id() + 1; });
 }
 
 bool GRenderWindow::event(QEvent* event) {

src/yuzu/bootmanager.h

@@ -11,6 +11,7 @@
 
 #include <QImage>
 #include <QThread>
+#include <QTouchEvent>
 #include <QWidget>
 #include <QWindow>
 
@@ -21,7 +22,6 @@
 class GRenderWindow;
 class GMainWindow;
 class QKeyEvent;
-class QTouchEvent;
 class QStringList;
 
 namespace InputCommon {
@@ -191,6 +191,10 @@ private:
     void TouchUpdateEvent(const QTouchEvent* event);
     void TouchEndEvent();
 
+    bool TouchStart(const QTouchEvent::TouchPoint& touch_point);
+    bool TouchUpdate(const QTouchEvent::TouchPoint& touch_point);
+    bool TouchExist(std::size_t id, const QList<QTouchEvent::TouchPoint>& touch_points) const;
+
     void OnMinimalClientAreaChangeRequest(std::pair<u32, u32> minimal_size) override;
 
     bool InitializeOpenGL();
@@ -215,6 +219,8 @@ private:
 
     bool first_frame = false;
 
+    std::array<std::size_t, 16> touch_ids{};
+
 protected:
     void showEvent(QShowEvent* event) override;
     bool eventFilter(QObject* object, QEvent* event) override;

src/yuzu_cmd/emu_window/emu_window_sdl2.cpp

@@ -29,16 +29,16 @@ EmuWindow_SDL2::~EmuWindow_SDL2() {
 }
 
 void EmuWindow_SDL2::OnMouseMotion(s32 x, s32 y) {
-    TouchMoved((unsigned)std::max(x, 0), (unsigned)std::max(y, 0));
+    TouchMoved((unsigned)std::max(x, 0), (unsigned)std::max(y, 0), 0);
     input_subsystem->GetMouse()->MouseMove(x, y);
 }
 
 void EmuWindow_SDL2::OnMouseButton(u32 button, u8 state, s32 x, s32 y) {
     if (button == SDL_BUTTON_LEFT) {
         if (state == SDL_PRESSED) {
-            TouchPressed((unsigned)std::max(x, 0), (unsigned)std::max(y, 0));
+            TouchPressed((unsigned)std::max(x, 0), (unsigned)std::max(y, 0), 0);
         } else {
-            TouchReleased();
+            TouchReleased(0);
         }
     } else if (button == SDL_BUTTON_RIGHT) {
         if (state == SDL_PRESSED) {
@@ -66,16 +66,16 @@ void EmuWindow_SDL2::OnFingerDown(float x, float y) {
     // 3DS does
 
     const auto [px, py] = TouchToPixelPos(x, y);
-    TouchPressed(px, py);
+    TouchPressed(px, py, 0);
 }
 
 void EmuWindow_SDL2::OnFingerMotion(float x, float y) {
     const auto [px, py] = TouchToPixelPos(x, y);
-    TouchMoved(px, py);
+    TouchMoved(px, py, 0);
 }
 
 void EmuWindow_SDL2::OnFingerUp() {
-    TouchReleased();
+    TouchReleased(0);
 }
 
 void EmuWindow_SDL2::OnKeyEvent(int key, u8 state) {