Fix mojibake when double byte text is written under a single byte font encoding

MsgReader/Rtf/Document.cs

@@ -48,6 +48,11 @@ internal class Document
     ///     The default rtf encoding
     /// </summary>
     private Encoding _defaultEncoding = Encoding.Default;
+
+    /// <summary>
+    ///     Set to <c>true</c> when the document declared its code page with <c>\ansicpg</c>
+    /// </summary>
+    private bool _documentCodePageDeclared;
 
     /// <summary>
     ///     Current runtime encoding
@@ -155,6 +160,7 @@ public void DeEncapsulateHtmlFromRtf(string rtf)
                         case Consts.Ansicpg:
                             // Read default encoding
                             _defaultEncoding = Font.EncodingFromCodePage(reader.Parameter);
+                            _documentCodePageDeclared = true;
                             break;
 
                         case Consts.Deff:
@@ -401,6 +407,37 @@ public void DeEncapsulateHtmlFromRtf(string rtf)
     }
     #endregion
 
+    #region FallbackEncoding
+    /// <summary>
+    ///     Returns the encoding to fall back on when the charset detection did not produce a usable result.
+    /// </summary>
+    /// <remarks>
+    ///     <see cref="TryDecode" /> is only called when the <see cref="FontTable" /> contains mixed encodings and a
+    ///     high byte (>= 0x80) is read while a <b>single byte</b> font encoding is active. In that situation the
+    ///     current font encoding is by definition suspect, so falling back to it cannot recover the text.<br /><br />
+    ///     When the document code page (<c>\ansicpg</c>) is a multi byte code page it can represent the double byte
+    ///     characters that the single byte font encoding cannot, so it is the better fallback. A typical example is a
+    ///     Korean (<c>\ansicpg949</c>) or Japanese (<c>\ansicpg932</c>) mail where part of the text is written under a
+    ///     font that declares <c>\fcharset0</c> (ANSI). Decoding those bytes with the ANSI code page silently produces
+    ///     mojibake because a single byte code page maps nearly every byte.<br /><br />
+    ///     When both encodings are single byte the document code page has no advantage, so the runtime encoding is
+    ///     kept. The document code page is also only trusted when the document actually declared one with
+    ///     <c>\ansicpg</c>, because <see cref="_defaultEncoding" /> otherwise still holds its
+    ///     <see cref="Encoding.Default" /> seed value (UTF-8 on .NET Core), which is multi byte and would wrongly win.
+    /// </remarks>
+    private Encoding FallbackEncoding
+    {
+        get
+        {
+            if (!_documentCodePageDeclared || _defaultEncoding == null || _defaultEncoding.IsSingleByte || !RuntimeEncoding.IsSingleByte)
+                return RuntimeEncoding;
+
+            Logger.WriteToLog($"The runtime encoding '{RuntimeEncoding.WebName}' is single byte while the document code page '{_defaultEncoding.WebName}' is multi byte, using the document code page instead");
+            return _defaultEncoding;
+        }
+    }
+    #endregion
+
     #region TryDecode
     /// <summary>
     ///     Tries to decode the byte buffer
@@ -426,16 +463,16 @@ private string TryDecode(ByteBuffer byteBuffer)
                     return byteBuffer.GetString(detectionResult.Detected.Encoding);
                 }
 
-                Logger.WriteToLog("Could not find the detected encoding in the font table, falling back to the RunTimeEncoding");
-                return byteBuffer.GetString(RuntimeEncoding);
+                Logger.WriteToLog("Could not find the detected encoding in the font table, falling back to the fallback encoding");
+                return byteBuffer.GetString(FallbackEncoding);
             }
 
-            Logger.WriteToLog($"Ignored detected encoding because it was not above the threshold of '{CharsetDetectionEncodingConfidenceLevel} using encoding '{RuntimeEncoding}' instead");
+            Logger.WriteToLog($"Ignored detected encoding because it was not above the threshold of '{CharsetDetectionEncodingConfidenceLevel}'");
         }
         else
             Logger.WriteToLog("No encoding detected");
 
-        return byteBuffer.GetString(RuntimeEncoding);
+        return byteBuffer.GetString(FallbackEncoding);
     }
     #endregion
 

MsgReaderTests/RtfDocumentTests.cs

@@ -64,6 +64,64 @@ public void Issue347()
             Deal("SampleFiles/rtf/Issue347.html", rtfDomDocument.HtmlContent);
         }
 
+        /// <summary>
+        ///     A double byte document code page (\ansicpg949, Korean) combined with a font table that mixes
+        ///     \fcharset129 (Hangul) and \fcharset0 (ANSI). The Korean text is written under the \fcharset0 font,
+        ///     which is what Outlook does in practice.
+        /// </summary>
+        /// <remarks>
+        ///     The mixed font table plus a high byte under a single byte font encoding routes these bytes through
+        ///     TryDecode. Charset detection cannot identify two bytes, so the fallback encoding decides the result.
+        ///     Falling back to the single byte font encoding (cp1252) silently turns each double byte Korean
+        ///     character into two Latin characters, so the document code page has to win here.
+        /// </remarks>
+        [TestMethod]
+        public void MixedCharsetFontsWithDoubleByteDocumentCodePage()
+        {
+            var rtfDomDocument = new Document();
+            rtfDomDocument.DeEncapsulateHtmlFromRtf(
+                "{\\rtf1\\ansi\\ansicpg949\\fromhtml1" +
+                "{\\fonttbl{\\f0\\fnil\\fcharset129 Malgun Gothic;}{\\f1\\fnil\\fcharset0 Calibri;}}" +
+                "\\htmlrtf{\\htmlrtf0\\f1 \\'be\\'c8\\'b3\\'e7\\'c7\\'cf\\'bc\\'bc\\'bf\\'e4}}");
+            Assert.AreEqual(expected: "안녕하세요", actual: rtfDomDocument.HtmlContent, ignoreCase: false);
+        }
+
+        /// <summary>
+        ///     A single byte document code page must not override the font encoding, so that a font declaring a
+        ///     different single byte charset than the document keeps winning.
+        /// </summary>
+        [TestMethod]
+        public void MixedCharsetFontsWithSingleByteDocumentCodePageKeepsFontEncoding()
+        {
+            var rtfDomDocument = new Document();
+            rtfDomDocument.DeEncapsulateHtmlFromRtf(
+                "{\\rtf1\\ansi\\ansicpg1252\\fromhtml1" +
+                "{\\fonttbl{\\f0\\fnil\\fcharset204 Arial;}{\\f1\\fnil\\fcharset129 Malgun Gothic;}}" +
+                "\\htmlrtf{\\htmlrtf0\\f0 \\'cf\\'f0\\'e8\\'e2\\'e5\\'f2}}");
+            Assert.AreEqual(expected: "Привет", actual: rtfDomDocument.HtmlContent, ignoreCase: false);
+        }
+
+        /// <summary>
+        ///     Issue 529. A Japanese (\ansicpg932) mail where a &amp;nbsp; emits a \fcharset0 font inside its own
+        ///     group, and the Japanese text on the following line inherits that single byte encoding.
+        /// </summary>
+        /// <remarks>
+        ///     This is the structure Word produces for a NO-BREAK SPACE: {\f1\'a0} switches to a single byte ANSI
+        ///     font, and the double byte Shift-JIS text that follows carries no font of its own. Decoding it with
+        ///     cp1252 turns each character into two Latin characters, so \'82\'a0 becomes U+201A U+00A0 instead of
+        ///     U+3042.
+        /// </remarks>
+        [TestMethod]
+        public void MixedCharsetFontsWithNoBreakSpaceBeforeDoubleByteText()
+        {
+            var rtfDomDocument = new Document();
+            rtfDomDocument.DeEncapsulateHtmlFromRtf(
+                "{\\rtf1\\ansi\\ansicpg932\\fromhtml1" +
+                "{\\fonttbl{\\f0\\fswiss\\fcharset128 MS PGothic;}{\\f1\\fmodern\\fcharset0 Courier New;}}" +
+                "\\f0\\htmlrtf {\\f1\\'a0}\\htmlrtf0 \\htmlrtf {\\htmlrtf0 \\'82\\'a0\\'82\\'a0\\'82\\'a0}}");
+            Assert.AreEqual(expected: "あああ", actual: rtfDomDocument.HtmlContent, ignoreCase: false);
+        }
+
         private static void Deal(string filePath, string rtf)
         {
             Assert.AreEqual(expected: File.ReadAllText(filePath), actual: rtf);