Add support for the S8C1T/S7C1T escape sequences (#17945)

This PR adds support for the `S8C1T` and `S7C1T` commands, which enable
an application to choose whether the terminal should use C1 controls
when sending key sequences and query responses.

This also updates the `DOCS` command to set both the input and output
code pages. So when switched to ISO2022 mode, the C1 controls will be
transmitted as 8-bit, which is what legacy systems would be expecting.

## Detailed Description of the Pull Request / Additional comments

While adding the input code page support, I also reworked the way we
handle the code page reset in `RIS`. In the original implementation we
saved the active code page when the `DOCS` sequence was first used, and
that would become the default value for a reset.

With this PR I'm now saving the code pages whenever `SetConsoleCP` or
`SetConsoleOutputCP` is called, so those APIs now control what the
default values will be. This feels more consistent than the previous
approach. And this is how WSL sets its initial code page to UTF-8.

## Validation Steps Performed

I've added a couple of unit tests that check one of each applicable C1
control in the key sequences and query reports.

I also built myself a code page aware telnet client so I could log into
WSL in 8-bit mode, and confirmed that the C1 transmissions are working
as expected in vttest.

Closes #17931
Tests added/passed

src/cascadia/TerminalCore/Terminal.hpp

@@ -139,8 +139,10 @@ class Microsoft::Terminal::Core::Terminal final :
     void SetWindowTitle(const std::wstring_view title) override;
     CursorType GetUserDefaultCursorStyle() const noexcept override;
     bool ResizeWindow(const til::CoordType width, const til::CoordType height) override;
-    void SetConsoleOutputCP(const unsigned int codepage) noexcept override;
-    unsigned int GetConsoleOutputCP() const noexcept override;
+    void SetCodePage(const unsigned int codepage) noexcept override;
+    void ResetCodePage() noexcept override;
+    unsigned int GetOutputCodePage() const noexcept override;
+    unsigned int GetInputCodePage() const noexcept override;
     void CopyToClipboard(wil::zwstring_view content) override;
     void SetTaskbarProgress(const ::Microsoft::Console::VirtualTerminal::DispatchTypes::TaskbarState state, const size_t progress) override;
     void SetWorkingDirectory(std::wstring_view uri) override;

src/cascadia/TerminalCore/TerminalApi.cpp

@@ -116,14 +116,25 @@ bool Terminal::ResizeWindow(const til::CoordType width, const til::CoordType hei
     return false;
 }
 
-void Terminal::SetConsoleOutputCP(const unsigned int /*codepage*/) noexcept
+void Terminal::SetCodePage(const unsigned int /*codepage*/) noexcept
 {
-    // TODO: This will be needed to support 8-bit charsets and DOCS sequences.
+    // Code pages are dealt with in ConHost, so this isn't needed.
 }
 
-unsigned int Terminal::GetConsoleOutputCP() const noexcept
+void Terminal::ResetCodePage() noexcept
 {
-    // TODO: See SetConsoleOutputCP above.
+    // There is nothing to reset, since the code page never changes.
+}
+
+unsigned int Terminal::GetOutputCodePage() const noexcept
+{
+    // See above. The code page is always UTF-8.
+    return CP_UTF8;
+}
+
+unsigned int Terminal::GetInputCodePage() const noexcept
+{
+    // See above. The code page is always UTF-8.
     return CP_UTF8;
 }
 

src/host/getset.cpp

@@ -1140,7 +1140,6 @@ void ApiRoutines::GetLargestConsoleWindowSizeImpl(const SCREEN_INFORMATION& cont
     {
         // Set new code page
         gci.OutputCP = codepage;
-
         SetConsoleCPInfo(TRUE);
     }
 
@@ -1157,13 +1156,36 @@ void ApiRoutines::GetLargestConsoleWindowSizeImpl(const SCREEN_INFORMATION& cont
 {
     try
     {
+        auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
         LockConsole();
         auto Unlock = wil::scope_exit([&] { UnlockConsole(); });
-        return DoSrvSetConsoleOutputCodePage(codepage);
+        RETURN_IF_FAILED(DoSrvSetConsoleOutputCodePage(codepage));
+        // Setting the code page via the API also updates the default value.
+        // This is how the initial code page is set to UTF-8 in a WSL shell.
+        gci.DefaultOutputCP = codepage;
+        return S_OK;
     }
     CATCH_RETURN();
 }
 
+[[nodiscard]] HRESULT DoSrvSetConsoleInputCodePage(const unsigned int codepage)
+{
+    auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
+
+    // Return if it's not known as a valid codepage ID.
+    RETURN_HR_IF(E_INVALIDARG, !(IsValidCodePage(codepage)));
+
+    // Do nothing if no change.
+    if (gci.CP != codepage)
+    {
+        // Set new code page
+        gci.CP = codepage;
+        SetConsoleCPInfo(FALSE);
+    }
+
+    return S_OK;
+}
+
 // Routine Description:
 // - Sets the codepage used for translating text when calling A versions of functions affecting the input buffer.
 // Arguments:
@@ -1177,19 +1199,10 @@ void ApiRoutines::GetLargestConsoleWindowSizeImpl(const SCREEN_INFORMATION& cont
         auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
         LockConsole();
         auto Unlock = wil::scope_exit([&] { UnlockConsole(); });
-
-        // Return if it's not known as a valid codepage ID.
-        RETURN_HR_IF(E_INVALIDARG, !(IsValidCodePage(codepage)));
-
-        // Do nothing if no change.
-        if (gci.CP != codepage)
-        {
-            // Set new code page
-            gci.CP = codepage;
-
-            SetConsoleCPInfo(FALSE);
-        }
-
+        RETURN_IF_FAILED(DoSrvSetConsoleInputCodePage(codepage));
+        // Setting the code page via the API also updates the default value.
+        // This is how the initial code page is set to UTF-8 in a WSL shell.
+        gci.DefaultCP = codepage;
         return S_OK;
     }
     CATCH_RETURN();

src/host/getset.h

@@ -19,3 +19,4 @@ Revision History:
 class SCREEN_INFORMATION;
 
 [[nodiscard]] HRESULT DoSrvSetConsoleOutputCodePage(const unsigned int codepage);
+[[nodiscard]] HRESULT DoSrvSetConsoleInputCodePage(const unsigned int codepage);

src/host/output.cpp

@@ -35,6 +35,8 @@ using namespace Microsoft::Console::Interactivity;
     // codepage by console.cpl or shell32. The default codepage is OEMCP.
     gci.CP = gci.GetCodePage();
     gci.OutputCP = gci.GetCodePage();
+    gci.DefaultCP = gci.GetCodePage();
+    gci.DefaultOutputCP = gci.GetCodePage();
 
     gci.Flags |= CONSOLE_USE_PRIVATE_FLAGS;
 

src/host/outputStream.cpp

@@ -221,27 +221,52 @@ void ConhostInternalGetSet::ShowWindow(bool showOrHide)
 }
 
 // Routine Description:
-// - Connects the SetConsoleOutputCP API call directly into our Driver Message servicing call inside Conhost.exe
+// - Set the code page used for translating text when calling A versions of I/O functions.
 // Arguments:
-// - codepage - the new output codepage of the console.
+// - codepage - the new code page of the console.
 // Return Value:
 // - <none>
-void ConhostInternalGetSet::SetConsoleOutputCP(const unsigned int codepage)
+void ConhostInternalGetSet::SetCodePage(const unsigned int codepage)
 {
-    THROW_IF_FAILED(DoSrvSetConsoleOutputCodePage(codepage));
+    LOG_IF_FAILED(DoSrvSetConsoleOutputCodePage(codepage));
+    LOG_IF_FAILED(DoSrvSetConsoleInputCodePage(codepage));
 }
 
 // Routine Description:
-// - Gets the codepage used for translating text when calling A versions of functions affecting the output buffer.
+// - Reset the code pages to their default values.
 // Arguments:
 // - <none>
 // Return Value:
-// - the outputCP of the console.
-unsigned int ConhostInternalGetSet::GetConsoleOutputCP() const
+// - <none>
+void ConhostInternalGetSet::ResetCodePage()
+{
+    const auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
+    LOG_IF_FAILED(DoSrvSetConsoleOutputCodePage(gci.DefaultOutputCP));
+    LOG_IF_FAILED(DoSrvSetConsoleInputCodePage(gci.DefaultCP));
+}
+
+// Routine Description:
+// - Gets the code page used for translating text when calling A versions of output functions.
+// Arguments:
+// - <none>
+// Return Value:
+// - the output code page of the console.
+unsigned int ConhostInternalGetSet::GetOutputCodePage() const
 {
     return ServiceLocator::LocateGlobals().getConsoleInformation().OutputCP;
 }
 
+// Routine Description:
+// - Gets the code page used for translating text when calling A versions of input functions.
+// Arguments:
+// - <none>
+// Return Value:
+// - the input code page of the console.
+unsigned int ConhostInternalGetSet::GetInputCodePage() const
+{
+    return ServiceLocator::LocateGlobals().getConsoleInformation().CP;
+}
+
 // Routine Description:
 // - Copies the given content to the clipboard.
 // Arguments:

src/host/outputStream.hpp

@@ -53,8 +53,10 @@ class ConhostInternalGetSet final : public Microsoft::Console::VirtualTerminal::
 
     bool ResizeWindow(const til::CoordType width, const til::CoordType height) override;
 
-    void SetConsoleOutputCP(const unsigned int codepage) override;
-    unsigned int GetConsoleOutputCP() const override;
+    void SetCodePage(const unsigned int codepage) override;
+    void ResetCodePage() override;
+    unsigned int GetOutputCodePage() const override;
+    unsigned int GetInputCodePage() const override;
 
     void CopyToClipboard(const wil::zwstring_view content) override;
     void SetTaskbarProgress(const ::Microsoft::Console::VirtualTerminal::DispatchTypes::TaskbarState state, const size_t progress) override;

src/host/server.h

@@ -90,6 +90,9 @@ class CONSOLE_INFORMATION :
     // the following fields are used for ansi-unicode translation
     UINT CP = 0;
     UINT OutputCP = 0;
+    // the VT RIS sequence uses these default values to reset the code pages
+    UINT DefaultCP = 0;
+    UINT DefaultOutputCP = 0;
 
     ULONG CtrlFlags = 0; // indicates outstanding ctrl requests
     ULONG LimitingProcessId = 0;

src/terminal/adapter/ITermDispatch.hpp

@@ -122,6 +122,7 @@ class Microsoft::Console::VirtualTerminal::ITermDispatch
     virtual void LockingShiftRight(const VTInt gsetNumber) = 0; // LS1R, LS2R, LS3R
     virtual void SingleShift(const VTInt gsetNumber) = 0; // SS2, SS3
     virtual void AcceptC1Controls(const bool enabled) = 0; // DECAC1
+    virtual void SendC1Controls(const bool enabled) = 0; // S8C1T, S7C1T
     virtual void AnnounceCodeStructure(const VTInt ansiLevel) = 0; // ACS
 
     virtual void SoftReset() = 0; // DECSTR

src/terminal/adapter/ITerminalApi.hpp

@@ -72,8 +72,10 @@ namespace Microsoft::Console::VirtualTerminal
 
         virtual void ShowWindow(bool showOrHide) = 0;
 
-        virtual void SetConsoleOutputCP(const unsigned int codepage) = 0;
-        virtual unsigned int GetConsoleOutputCP() const = 0;
+        virtual void SetCodePage(const unsigned int codepage) = 0;
+        virtual void ResetCodePage() = 0;
+        virtual unsigned int GetOutputCodePage() const = 0;
+        virtual unsigned int GetInputCodePage() const = 0;
 
         virtual void CopyToClipboard(const wil::zwstring_view content) = 0;
         virtual void SetTaskbarProgress(const DispatchTypes::TaskbarState state, const size_t progress) = 0;

src/terminal/adapter/InteractDispatch.cpp

@@ -62,7 +62,7 @@ void InteractDispatch::WriteString(const std::wstring_view string)
 {
     if (!string.empty())
     {
-        const auto codepage = _api.GetConsoleOutputCP();
+        const auto codepage = _api.GetOutputCodePage();
         InputEventQueue keyEvents;
 
         for (const auto& wch : string)