From 2bfefed7a20dad792d40d00c20ececcc677b58da Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Tue, 15 Sep 2026 23:18:02 +0900 Subject: [PATCH] fix(indexer): give lang the same tag as content_ when a document brings its own language A page declaring was indexed with lang=zh_CN while its text went to content_zh-cn, and a Chinese page without the attribute got lang=zh-cn. The same language was stored two ways, and lang:zh-cn found only the pages whose language had been detected. FessXpathTransformer stores SystemHelper#normalizeLang of the attribute, which answers with the spelling of supported.languages, zh_CN. LanguageHelper#updateDocument resolves it to zh-cn for the content copy but left lang untouched. createScript, used by the label, click and favorite count updates, then copied the text to content_zh_CN, which no analyzer template matches. When a document already carries a lang string that resolves to a supported language, updateDocument now writes the resolved tag back. This also covers crawler.document.html.default.lang and crawler.document.file.default.lang configured as zh_CN, and pt_BR, en_IE and ckb_IQ. SystemHelper#normalizeLang is unchanged: its request-side values go through query.language.mapping and are never compared with the stored lang. --- .../codelibs/fess/helper/LanguageHelper.java | 6 ++++++ .../fess/helper/LanguageHelperTest.java | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/main/java/org/codelibs/fess/helper/LanguageHelper.java b/src/main/java/org/codelibs/fess/helper/LanguageHelper.java index 6f2bdb49d..0ba26d783 100644 --- a/src/main/java/org/codelibs/fess/helper/LanguageHelper.java +++ b/src/main/java/org/codelibs/fess/helper/LanguageHelper.java @@ -95,6 +95,12 @@ public void updateDocument(final Map doc) { if (language == null) { return; } + } else if (doc.get(fessConfig.getIndexFieldLang()) instanceof String) { + // A language the document brings with it -- , a configured default lang -- + // is written the way supported.languages writes it, zh_CN. The text is copied below to the field + // of the normalized tag, content_zh-cn, so lang carries that tag too, as it does for a detected + // language; otherwise lang:zh-cn does not find the document. + doc.put(fessConfig.getIndexFieldLang(), language); } for (final String f : langFields) { diff --git a/src/test/java/org/codelibs/fess/helper/LanguageHelperTest.java b/src/test/java/org/codelibs/fess/helper/LanguageHelperTest.java index 79e3620bd..c7d922067 100644 --- a/src/test/java/org/codelibs/fess/helper/LanguageHelperTest.java +++ b/src/test/java/org/codelibs/fess/helper/LanguageHelperTest.java @@ -123,6 +123,25 @@ public void test_updateDocument_skipExistingLangFields() { assertEquals("コンテンツ", doc.get("content_ja")); } + @Test + public void test_updateDocument_normalizesExistingLang() { + // reaches the document as zh_CN, the spelling supported.languages uses. The + // text is copied to content_zh-cn, so lang has to carry the same tag a detected document gets, + // or lang:zh-cn does not find the page. + languageHelper.supportedLanguages = new String[] { "ja", "en", "zh_CN", "zh_TW" }; + Map doc = new HashMap<>(); + doc.put("lang", "zh_CN"); + doc.put("title", "标题"); + doc.put("content", "内容"); + + languageHelper.updateDocument(doc); + + assertEquals("zh-cn", doc.get("lang")); + assertEquals("标题", doc.get("title_zh-cn")); + assertEquals("内容", doc.get("content_zh-cn")); + assertNull(doc.get("content_zh_CN")); + } + @Test public void test_detectLanguage_blank() { assertNull(languageHelper.detectLanguage(null));