enhance TTS (#1054)

* fix: read by paragraph and empty string

* add voice picker modal

* add tts voice picker for preview

* fix: tts for bionic reading

* stop speak when get out of reader

* search voice

* temporary fix: original font

* fix: start & resume

* fix: close context menu on copy

android/app/src/main/assets/css/index.css

@@ -307,7 +307,7 @@ td {
   text-align: right;
 }
 
-tts.highlight {
+.highlight {
   color: var(--theme-onSecondary);
   background-color: var(--theme-secondary);
 }

android/app/src/main/assets/js/index.js

@@ -101,14 +101,19 @@ class Reader {
       '--readerSettings-fontFamily',
       settings.fontFamily,
     );
-    new FontFace(
-      settings.fontFamily,
-      'url("file:///android_asset/fonts/' + settings.fontFamily + '.ttf")',
-    )
-      .load()
-      .then(function (loadedFont) {
-        document.fonts.add(loadedFont);
-      });
+    if (settings.fontFamily) {
+      new FontFace(
+        settings.fontFamily,
+        'url("file:///android_asset/fonts/' + settings.fontFamily + '.ttf")',
+      )
+        .load()
+        .then(function (loadedFont) {
+          document.fonts.add(loadedFont);
+        });
+    } else {
+      // have no affect with a font declared in head
+      document.fonts.forEach(fontFace => document.fonts.delete(fontFace));
+    }
   };
   updateGeneralSettings = settings => {
     this.showScrollPercentage = settings.showScrollPercentage;
@@ -314,12 +319,15 @@ class SwipeHandler {
 /**
  * @type {import('./type').TextToSpeech}
  */
+
+// Walk through all leave -> convert each leaf to list of tts elements -> read through all tts elements
 class TextToSpeech {
   constructor(reader) {
     this.reader = reader;
-    (this.leaf = this.reader.chapter), (this.TTSWrapper = null); // swap these 2 elements
-    this.TTSEle = null;
+    this.readableNodeNames = ['#text', 'B', 'I', 'SPAN', 'EM', 'BR', 'STRONG'];
+    this.leaf = this.reader.chapter;
     this.reading = false;
+    this.started = false;
     this.$ = document.getElementById('TTS-Controller');
     this.$.classList.add('d-none');
     this.$.onclick = () => {
@@ -332,9 +340,11 @@ class TextToSpeech {
   }
 
   start = () => {
+    this.started = false;
     this.$.innerHTML = `<button>${pauseIcon}</button>`;
     this.$.classList.remove('d-none');
     this.stop();
+    this.leaf = this.reader.chapter;
     this.next();
   };
 
@@ -343,11 +353,11 @@ class TextToSpeech {
     this.$.classList.remove('d-none');
     this.stop();
     if (this.reader.selection.type === 'Range') {
-      if (this.leaf && this.TTSWrapper) {
-        this.TTSWrapper.replaceWith(this.leaf);
-      }
       this.leaf = this.reader.selection.anchorNode;
-      this.makeLeafSpeakable();
+      while (this.readableNodeNames.includes(this.leaf.nodeName)) {
+        this.leaf = this.leaf.parentNode;
+      }
+      this.reader.selection.removeAllRanges();
       this.speak();
     } else {
       this.next();
@@ -359,77 +369,58 @@ class TextToSpeech {
     this.next();
   };
 
-  findLeaf() {
-    while (this.leaf.firstChild) {
-      this.leaf = this.leaf.firstChild;
-    }
-  }
-
-  findNextLeaf() {
-    if (this.reader.chapter.isSameNode(this.leaf)) {
-      this.findLeaf();
-    } else if (this.leaf.nextSibling) {
-      this.leaf = this.leaf.nextSibling;
-      this.findLeaf();
-    } else {
-      this.leaf = this.leaf.parentNode;
-      if (this.reader.chapter.isSameNode(this.leaf)) {
-        return;
-      }
-      this.findNextLeaf();
-    }
-  }
-
   readable() {
-    return (
-      this.leaf.nodeName === '#text' && /[^\s\n,.?!;":“”]/.test(this.leaf.data)
-    );
+    if (
+      this.leaf.nodeName !== 'SPAN' &&
+      this.readableNodeNames.includes(this.leaf.nodeName)
+    ) {
+      return false;
+    }
+    if (!this.leaf.hasChildNodes()) {
+      return false;
+    }
+    const a = [];
+    for (let i = 0; i < this.leaf.childNodes.length; i++) {
+      if (
+        !this.readableNodeNames.includes(this.leaf.childNodes.item(i).nodeName)
+      ) {
+        return false;
+      }
+    }
+    return true;
   }
 
   findNextTextNode() {
-    if (this.leaf && this.TTSWrapper) {
-      this.TTSWrapper.replaceWith(this.leaf);
-      this.TTSWrapper = null;
-    }
     do {
-      this.findNextLeaf();
-    } while (!this.readable() && !this.reader.chapter.isSameNode(this.leaf));
+      if (this.reader.chapter.isSameNode(this.leaf)) {
+        this.leaf = this.leaf.firstElementChild;
+      } else {
+        if (this.leaf.firstElementChild) {
+          this.leaf = this.leaf.firstElementChild;
+        } else if (this.leaf.nextElementSibling) {
+          this.leaf = this.leaf.nextElementSibling;
+        } else {
+          this.leaf = this.leaf.parentElement;
+        }
+      }
+    } while (!this.readable());
     if (this.reader.chapter.isSameNode(this.leaf)) {
       return;
     }
-    this.makeLeafSpeakable();
-  }
-
-  makeLeafSpeakable() {
-    this.TTSWrapper = document.createElement('tts-wrapper');
-    this.TTSWrapper.innerHTML = this.leaf.data.replace(
-      /([^\n,.?!;":“”]+)/g,
-      matched => {
-        if (matched.trim().length) {
-          return `<tts>${matched}</tts>`;
-        }
-        return matched;
-      },
-    );
-    this.TTSEle = this.TTSWrapper.firstElementChild;
-    this.leaf.replaceWith(this.TTSWrapper);
   }
 
   next = () => {
     try {
-      if (this.TTSEle) {
-        this.TTSEle.classList.remove('highlight');
-      }
+      this.leaf?.classList?.remove('highlight');
       if (this.reading) {
-        if (this.TTSEle && this.TTSEle.nextElementSibling) {
-          this.TTSEle = this.TTSEle.nextElementSibling;
+        if (this.leaf.nextElementSibling) {
+          this.leaf = this.leaf.nextElementSibling;
         } else {
           this.findNextTextNode();
         }
-      } else {
-        if (!this.TTSEle) {
-          this.findNextTextNode();
-        }
+      } else if (!this.started) {
+        this.findNextTextNode();
+        this.started = true;
       }
       this.speak();
     } catch (e) {
@@ -438,15 +429,10 @@ class TextToSpeech {
   };
 
   stop = () => {
-    this.$.classList.add('d-none');
-    this.reading = false;
-    if (this.leaf && this.TTSWrapper) {
-      this.TTSWrapper.replaceWith(this.leaf);
-      this.TTSWrapper = null;
-      this.TTSEle = null;
-      this.leaf = this.reader.chapter;
-    }
     this.reader.post({ type: 'stop-speak' });
+    this.$.classList.add('d-none');
+    this.leaf?.classList?.remove('highlight');
+    this.reading = false;
   };
 
   pause = () => {
@@ -455,15 +441,13 @@ class TextToSpeech {
     this.reader.post({ type: 'stop-speak' });
   };
 
-  started = () => this.TTSWrapper !== null;
-
   speak = () => {
     this.reading = true;
     if (this.reader.chapter.isSameNode(this.leaf)) {
       return;
     }
-    this.TTSEle.classList.add('highlight');
-    this.reader.post({ type: 'speak', data: this.TTSEle?.innerText.trim() });
+    this.leaf.classList.add('highlight');
+    this.reader.post({ type: 'speak', data: this.leaf?.innerText });
   };
 }
 
@@ -481,22 +465,23 @@ class ContextMenu {
         name: 'Start Reading',
         icon: volumeIcon,
         action: () => {
-          tts.start();
           this.closeMenu();
+          tts.start();
         },
       }),
       START_HERE: this.renderItem({
         name: 'Start Here',
         icon: selectVolumeIcon,
         action: () => {
-          tts.startHere();
           this.closeMenu();
+          tts.startHere();
         },
       }),
       COPY: this.renderItem({
         name: 'Copy',
         icon: copyIcon,
         action: () => {
+          this.closeMenu();
           this.reader.post({
             type: 'copy',
             data: this.reader.selection.toString(),

android/app/src/main/assets/js/type.d.ts

@@ -30,10 +30,9 @@ interface Reader {
 
 interface TextToSpeech {
   reader: Reader;
-  leaf: HTMLElement;
-  TTSWrapper: HTMLElement;
-  TTSEle: HTMLElement;
+  leaf: HTMLUnknownElement;
   reading: boolean;
+  readableTags: string[];
   start: () => void;
   startHere: () => void;
   resume: () => void;

src/hooks/persisted/useSettings.ts

@@ -4,6 +4,7 @@ import {
   LibrarySortOrder,
 } from '@screens/library/constants/constants';
 import { useMMKVObject } from 'react-native-mmkv';
+import { Voice } from 'expo-speech';
 
 export const APP_SETTINGS = 'APP_SETTINGS';
 export const BROWSE_SETTINGS = 'BROWSE_SETTINGS';
@@ -100,6 +101,11 @@ export interface ChapterReaderSettings {
   customCSS: string;
   customJS: string;
   customThemes: ReaderTheme[];
+  tts?: {
+    voice?: Voice;
+    rate?: number;
+    pitch?: number;
+  };
 }
 
 const initialAppSettings: AppSettings = {
@@ -173,6 +179,10 @@ export const initialChapterReaderSettings: ChapterReaderSettings = {
   customCSS: '',
   customJS: '',
   customThemes: [],
+  tts: {
+    rate: 1,
+    pitch: 1,
+  },
 };
 
 export const useAppSettings = () => {

src/screens/reader/ReaderScreen.tsx

@@ -11,6 +11,7 @@ import {
   DrawerLayoutAndroid,
 } from 'react-native';
 import * as RNFS from 'react-native-fs';
+import * as Speech from 'expo-speech';
 
 import VolumeButtonListener from '@native/volumeButtonListener';
 
@@ -158,6 +159,7 @@ export const ChapterContent = ({
       VolumeButtonListener.disconnect();
       emmiter.current.removeAllListeners('VolumeUp');
       emmiter.current.removeAllListeners('VolumeDown');
+      Speech.stop();
     };
   }, [useVolumeButtons, chapter]);
 

src/screens/reader/components/WebViewReader.tsx

@@ -168,7 +168,12 @@ const WebViewReader: FC<WebViewReaderProps> = props => {
                 onDone() {
                   webViewRef.current?.injectJavaScript('tts.next?.()');
                 },
+                voice: readerSettings.tts?.voice?.identifier,
+                pitch: readerSettings.tts?.pitch || 1,
+                rate: readerSettings.tts?.rate || 1,
               });
+            } else {
+              webViewRef.current?.injectJavaScript('tts.next?.()');
             }
             break;
           case 'stop-speak':

src/screens/settings/SettingsReaderScreen/Modals/VoicePickerModal.tsx

@@ -0,0 +1,94 @@
+import React, { useEffect, useState } from 'react';
+import { StyleSheet } from 'react-native';
+
+import { Portal, Modal, overlay, TextInput } from 'react-native-paper';
+import { RadioButton } from '@components/RadioButton/RadioButton';
+
+import { useChapterReaderSettings, useTheme } from '@hooks/persisted';
+import { Voice, getAvailableVoicesAsync } from 'expo-speech';
+import { FlashList } from '@shopify/flash-list';
+
+interface VoicePickerModalProps {
+  visible: boolean;
+  onDismiss: () => void;
+}
+
+const VoicePickerModal: React.FC<VoicePickerModalProps> = ({
+  onDismiss,
+  visible,
+}) => {
+  const theme = useTheme();
+  const [voices, setVoices] = useState<Voice[]>([]);
+  const [searchedVoices, setSearchedVoices] = useState<Voice[]>([]);
+  const [searchText, setSearchText] = useState('');
+  const { setChapterReaderSettings, tts } = useChapterReaderSettings();
+  useEffect(() => {
+    getAvailableVoicesAsync().then(res => {
+      res.sort((a, b) => a.name.localeCompare(b.name));
+      setVoices([{ name: 'System', language: 'System' } as Voice, ...res]);
+    });
+  }, []);
+  return (
+    <Portal>
+      <Modal
+        visible={visible}
+        onDismiss={onDismiss}
+        contentContainerStyle={[
+          styles.containerStyle,
+          { backgroundColor: overlay(2, theme.surface) },
+        ]}
+      >
+        <FlashList
+          ListHeaderComponent={
+            <TextInput
+              mode="outlined"
+              underlineColor={theme.outline}
+              theme={{ colors: { ...theme } }}
+              onChangeText={text => {
+                setSearchText(text);
+                setSearchedVoices(
+                  voices.filter(voice =>
+                    voice.name
+                      .toLocaleLowerCase()
+                      .includes(text.toLocaleLowerCase()),
+                  ),
+                );
+              }}
+              value={searchText}
+              placeholder="Search voice"
+            />
+          }
+          ListHeaderComponentStyle={{ paddingHorizontal: 12 }}
+          data={searchText ? searchedVoices : voices}
+          extraData={tts?.voice}
+          renderItem={({ item }) => (
+            <RadioButton
+              key={item.identifier}
+              status={item.identifier === tts?.voice?.identifier}
+              onPress={() =>
+                setChapterReaderSettings({ tts: { ...tts, voice: item } })
+              }
+              label={item.name + ` (${item.language})`}
+              labelStyle={{ fontFamily: item.name }}
+              theme={theme}
+            />
+          )}
+          keyExtractor={item => item.identifier || 'system'}
+          estimatedItemSize={64}
+          removeClippedSubviews={true}
+        />
+      </Modal>
+    </Portal>
+  );
+};
+
+export default VoicePickerModal;
+
+const styles = StyleSheet.create({
+  containerStyle: {
+    paddingVertical: 24,
+    margin: 20,
+    borderRadius: 28,
+    flex: 1,
+  },
+});

src/screens/settings/SettingsReaderScreen/Settings/TextToSpeechSettings.tsx

@@ -0,0 +1,33 @@
+import { List } from '@components';
+import { useChapterReaderSettings, useTheme } from '@hooks/persisted';
+import React from 'react';
+import VoicePickerModal from '../Modals/VoicePickerModal';
+import { useBoolean } from '@hooks';
+import { Portal } from 'react-native-paper';
+
+export default function TextToSpeechSettings() {
+  const theme = useTheme();
+  const { tts } = useChapterReaderSettings();
+  const {
+    value: voiceModalVisible,
+    setTrue: showVoiceModal,
+    setFalse: hideVoiceModal,
+  } = useBoolean();
+  return (
+    <>
+      <List.SubHeader theme={theme}>Text to Speech</List.SubHeader>
+      <List.Item
+        title={'TTS voice'}
+        description={tts?.voice?.name || 'System'}
+        onPress={showVoiceModal}
+        theme={theme}
+      />
+      <Portal>
+        <VoicePickerModal
+          visible={voiceModalVisible}
+          onDismiss={hideVoiceModal}
+        />
+      </Portal>
+    </>
+  );
+}

src/screens/settings/SettingsReaderScreen/SettingsReaderScreen.tsx

@@ -25,6 +25,7 @@ import { useBatteryLevel } from 'react-native-device-info';
 import * as Speech from 'expo-speech';
 import * as Clipboard from 'expo-clipboard';
 import { showToast } from '@utils/showToast';
+import TextToSpeechSettings from './Settings/TextToSpeechSettings';
 
 export type TextAlignments =
   | 'left'
@@ -141,7 +142,12 @@ const SettingsReaderScreen = () => {
                     onDone() {
                       webViewRef.current?.injectJavaScript('tts.next?.()');
                     },
+                    voice: readerSettings.tts?.voice?.identifier,
+                    pitch: readerSettings.tts?.pitch || 1,
+                    rate: readerSettings.tts?.rate || 1,
                   });
+                } else {
+                  webViewRef.current?.injectJavaScript('tts.next?.()');
                 }
                 break;
               case 'stop-speak':
@@ -230,6 +236,8 @@ const SettingsReaderScreen = () => {
           <DisplaySettings />
           <List.Divider theme={theme} />
           <ReaderThemeSettings />
+          <List.Divider theme={theme} />
+          <TextToSpeechSettings />
         </View>
       </ScrollView>
     </>