diff -ruN include/teiplain.h include/teiplain.h --- include/teiplain.h 2026-03-15 17:32:39.218799807 +0000 +++ include/teiplain.h 2026-03-15 17:32:39.259799808 +0000 @@ -43,6 +43,7 @@ return new MyUserData(module, key); } virtual bool handleToken(SWBuf &buf, const char *token, BasicFilterUserData *userData); + virtual bool processStage(char stage, SWBuf &text, char *&from, BasicFilterUserData *userData); public: TEIPlain(); }; diff -ruN src/modules/filters/gbfplain.cpp src/modules/filters/gbfplain.cpp --- src/modules/filters/gbfplain.cpp 2026-03-15 17:34:25.159800118 +0000 +++ src/modules/filters/gbfplain.cpp 2026-03-15 17:33:17.203799918 +0000 @@ -109,7 +109,14 @@ token[tokpos+2] = 0; } } - else text.append(*from); + else { + // Normalize typographic apostrophe (U+2019, UTF-8: 0xE2 0x80 0x99) to standard apostrophe + if ((unsigned char)from[0] == 0xE2 && (unsigned char)from[1] == 0x80 && (unsigned char)from[2] == 0x99) { + text.append('\''); + from += 2; // skip 2 extra bytes (the 3rd will be skipped by the loop's ++from) + } + else text.append(*from); + } } return 0; } diff -ruN src/modules/filters/osisplain.cpp src/modules/filters/osisplain.cpp --- src/modules/filters/osisplain.cpp 2026-03-15 17:34:25.155800118 +0000 +++ src/modules/filters/osisplain.cpp 2026-03-15 17:33:10.922799900 +0000 @@ -83,6 +83,14 @@ if (stage == PRECHAR) { if ((unsigned)from[0] == 0xC2 && (unsigned)from[1] == 0xAD) return true; // skip soft hyphens + + // Normalize typographic apostrophe (U+2019, UTF-8: 0xE2 0x80 0x99) to standard apostrophe + // so that French searches work regardless of which apostrophe the user types + if ((unsigned char)from[0] == 0xE2 && (unsigned char)from[1] == 0x80 && (unsigned char)from[2] == 0x99) { + text.append('\''); + from += 2; // skip 2 extra bytes (the 3rd will be skipped by the main loop) + return true; + } } return false; } diff -ruN src/modules/filters/papyriplain.cpp src/modules/filters/papyriplain.cpp --- src/modules/filters/papyriplain.cpp 2026-03-15 17:34:25.164800118 +0000 +++ src/modules/filters/papyriplain.cpp 2026-03-15 17:33:24.788799941 +0000 @@ -79,7 +79,12 @@ } // if we've made it this far - text.append(*from); + // Normalize typographic apostrophe (U+2019, UTF-8: 0xE2 0x80 0x99) to standard apostrophe + if ((unsigned char)from[0] == 0xE2 && (unsigned char)from[1] == 0x80 && (unsigned char)from[2] == 0x99) { + text.append('\''); + from += 2; // skip 2 extra bytes (the 3rd will be skipped by the loop's ++from) + } + else text.append(*from); } return 0; diff -ruN src/modules/filters/teiplain.cpp src/modules/filters/teiplain.cpp --- src/modules/filters/teiplain.cpp 2026-03-15 17:32:39.194799807 +0000 +++ src/modules/filters/teiplain.cpp 2026-03-15 17:32:39.233799807 +0000 @@ -19,7 +19,6 @@ * General Public License for more details. * */ - #include #include #include @@ -29,26 +28,33 @@ TEIPlain::TEIPlain() { setTokenStart("<"); setTokenEnd(">"); - setEscapeStart("&"); setEscapeEnd(";"); - setEscapeStringCaseSensitive(true); - addEscapeStringSubstitute("amp", "&"); addEscapeStringSubstitute("apos", "'"); addEscapeStringSubstitute("lt", "<"); addEscapeStringSubstitute("gt", ">"); addEscapeStringSubstitute("quot", "\""); - setTokenCaseSensitive(true); } +bool TEIPlain::processStage(char stage, SWBuf &text, char *&from, BasicFilterUserData *userData) { + if (stage == PRECHAR) { + // Normalize typographic apostrophe (U+2019, UTF-8: 0xE2 0x80 0x99) to standard apostrophe + // so that French searches work regardless of which apostrophe the user types + if ((unsigned char)from[0] == 0xE2 && (unsigned char)from[1] == 0x80 && (unsigned char)from[2] == 0x99) { + text.append('\''); + from += 2; // skip 2 extra bytes (the 3rd will be skipped by the main loop) + return true; + } + } + return false; +} bool TEIPlain::handleToken(SWBuf &buf, const char *token, BasicFilterUserData *userData) { - // manually process if it wasn't a simple substitution + // manually process if it wasn't a simple substitution if (!substituteToken(buf, token)) { - //MyUserData *u = (MyUserData *)userData; XMLTag tag(token); //

paragraph tag @@ -93,7 +99,6 @@ //

else if (!strcmp(tag.getName(), "div")) { - if ((!tag.isEndTag()) && (!tag.isEmpty())) { buf.append("\n\n\n"); } @@ -116,7 +121,6 @@ else if (!strcmp(tag.getName(), "list")) { if ((!tag.isEndTag()) && (!tag.isEmpty())) { - buf += "\n"; } else if (tag.isEndTag()) { @@ -138,5 +142,4 @@ return true; } - SWORD_NAMESPACE_END diff -ruN src/modules/filters/thmlplain.cpp src/modules/filters/thmlplain.cpp --- src/modules/filters/thmlplain.cpp 2026-03-15 17:32:39.203799807 +0000 +++ src/modules/filters/thmlplain.cpp 2026-03-15 17:32:39.244799807 +0000 @@ -216,7 +216,14 @@ token[tokpos+2] = 0; } } - else text += *from; + else { + // Normalize typographic apostrophe (U+2019, UTF-8: 0xE2 0x80 0x99) to standard apostrophe + if ((unsigned char)from[0] == 0xE2 && (unsigned char)from[1] == 0x80 && (unsigned char)from[2] == 0x99) { + text += '\''; + from += 2; // skip 2 extra bytes (the 3rd will be skipped by the loop's ++from) + } + else text += *from; + } } orig = text; diff -ruN src/modules/filters/thmlwordjs.cpp src/modules/filters/thmlwordjs.cpp --- src/modules/filters/thmlwordjs.cpp 2026-03-15 17:32:39.213799807 +0000 +++ src/modules/filters/thmlwordjs.cpp 2026-03-15 17:32:39.254799807 +0000 @@ -2,7 +2,7 @@ * * thmlwordjs.cpp - SWFilter descendant to ??? * - * $Id: thmlwordjs.cpp 3909 2025-07-12 16:46:35Z scribe $ + * $Id: thmlwordjs.cpp 3808 2020-10-02 13:23:34Z scribe $ * * Copyright 2005-2013 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -151,7 +151,7 @@ text += token; text += '>'; if (needWordOut) { - char wstr[12]; + char wstr[16]; sprintf(wstr, "%03d", word-2); AttributeValue *wAttrs = &(module->getEntryAttributes()["Word"][wstr]); needWordOut = false; @@ -248,7 +248,7 @@ } } - char wstr[12]; + char wstr[16]; sprintf(wstr, "%03d", word-1); AttributeValue *wAttrs = &(module->getEntryAttributes()["Word"][wstr]); needWordOut = false; diff -ruN src/modules/swmodule.cpp src/modules/swmodule.cpp --- src/modules/swmodule.cpp 2026-03-15 17:32:39.185799807 +0000 +++ src/modules/swmodule.cpp 2026-03-15 17:32:39.223799807 +0000 @@ -398,6 +398,17 @@ listKey.clear(); SWBuf term = istr; + // Normalize typographic apostrophe (U+2019, UTF-8: 0xE2 0x80 0x99) to standard apostrophe + // so that French searches work regardless of which apostrophe the user types + { + std::string normalizedTerm = term.c_str(); + size_t pos = 0; + while ((pos = normalizedTerm.find("\xe2\x80\x99", pos)) != std::string::npos) { + normalizedTerm.replace(pos, 3, "'"); + pos += 1; + } + term = normalizedTerm.c_str(); + } bool includeComponents = false; // for entryAttrib e.g., /Lemma.1/ // this only works for 1 or 2 verses right now, and for some search types (regex and multi word). @@ -550,7 +561,12 @@ Xapian::Query q = queryParser.parse_query(istr); Xapian::Enquire enquire = Xapian::Enquire(database); #elif defined USELUCENE - q = QueryParser::parse((wchar_t *)utf8ToWChar(istr).getRawData(), _T("content"), &analyzer); + // Append a trailing space to work around a CLucene tokenizer bug where + // the last token is not finalized correctly without a following whitespace. + // This fixes: single +TERM searches, and some Unicode words (e.g. Greek LXX). + SWBuf istrFixed = istr; + istrFixed.append(' '); + q = QueryParser::parse((wchar_t *)utf8ToWChar(istrFixed.c_str()).getRawData(), _T("content"), &analyzer); #endif (*percent)(20, percentUserData);