aboutsummaryrefslogtreecommitdiffstats
path: root/native/jni/src/utils/char_utils.h
diff options
context:
space:
mode:
Diffstat (limited to 'native/jni/src/utils/char_utils.h')
-rw-r--r--native/jni/src/utils/char_utils.h4
1 files changed, 4 insertions, 0 deletions
diff --git a/native/jni/src/utils/char_utils.h b/native/jni/src/utils/char_utils.h
index f28ed5682..63786502b 100644
--- a/native/jni/src/utils/char_utils.h
+++ b/native/jni/src/utils/char_utils.h
@@ -98,6 +98,10 @@ class CharUtils {
// Beginning-of-Sentence.
static AK_FORCE_INLINE int attachBeginningOfSentenceMarker(int *const codePoints,
const int codePointCount, const int maxCodePoint) {
+ if (codePointCount > 0 && codePoints[0] == CODE_POINT_BEGINNING_OF_SENTENCE) {
+ // Marker has already been attached.
+ return codePointCount;
+ }
if (codePointCount >= maxCodePoint) {
// the code points cannot be marked as a Beginning-of-Sentence.
return 0;