Allow recovering from tty switching failures

If we fail to switch tty when showing the greeter, exit saying so then
switch back to tty 1 and try launching the Display again.

Fixes #1636

src/auth/Auth.h

@@ -86,6 +86,7 @@ namespace SDDM {
             HELPER_SESSION_ERROR,
             HELPER_OTHER_ERROR,
             HELPER_DISPLAYSERVER_ERROR,
+            HELPER_TTY_ERROR,
         };
         Q_ENUM(HelperExitStatus)
 

src/daemon/Display.cpp

@@ -52,6 +52,8 @@
 #include "VirtualTerminal.h"
 #include "WaylandDisplayServer.h"
 
+static int s_ttyFailures = 0;
+
 
 namespace SDDM {
     bool isTtyInUse(const QString &desiredTty) {
@@ -149,10 +151,17 @@ namespace SDDM {
         connect(this, &Display::loginFailed, m_socketServer, &SocketServer::loginFailed);
         connect(this, &Display::loginSucceeded, m_socketServer, &SocketServer::loginSucceeded);
 
-        connect(m_greeter, &Greeter::failed,
-                QCoreApplication::instance(), [] {
-                    QCoreApplication::instance()->exit(23);
-                });
+        connect(m_greeter, &Greeter::failed, this, &Display::stop);
+        connect(m_greeter, &Greeter::ttyFailed, this, [this] {
+            ++s_ttyFailures;
+            if (s_ttyFailures > 5) {
+                QCoreApplication::exit(23);
+            }
+            // It might be the case that we are trying a tty that has been taken over by a
+            // different process. In such a case, switch back to the initial one and try again.
+            VirtualTerminal::jumpToVt(1, true);
+            stop();
+        });
         connect(m_greeter, &Greeter::displayServerFailed, this, &Display::displayServerFailed);
     }
 

src/daemon/Greeter.cpp

@@ -32,6 +32,7 @@
 
 #include <QtCore/QDebug>
 #include <QtCore/QProcess>
+#include <VirtualTerminal.h>
 
 namespace SDDM {
     Greeter::Greeter(Display *parent)
@@ -312,9 +313,9 @@ namespace SDDM {
 
         if (status == Auth::HELPER_DISPLAYSERVER_ERROR) {
             Q_EMIT displayServerFailed();
-        }
-
-        if (status == Auth::HELPER_SESSION_ERROR) {
+        } else if (status == Auth::HELPER_TTY_ERROR) {
+            Q_EMIT ttyFailed();
+        } else if (status == Auth::HELPER_SESSION_ERROR) {
             Q_EMIT failed();
         }
     }

src/daemon/Greeter.h

@@ -62,6 +62,7 @@ namespace SDDM {
         void authError(const QString &message, Auth::Error error);
 
     signals:
+        void ttyFailed();
         void failed();
         void displayServerFailed();
 

src/helper/UserSession.cpp

@@ -180,7 +180,7 @@ namespace SDDM {
                 if (ioctl(STDIN_FILENO, TIOCSCTTY) < 0) {
                     const auto error = strerror(errno);
                     qCritical().nospace() << "Failed to take control of " << ttyString << " (" << QFileInfo(ttyString).owner() << "): " << error;
-                    exit(Auth::HELPER_OTHER_ERROR);
+                    exit(Auth::HELPER_TTY_ERROR);
                 }
             }