From e2d22a5b05a47b57e1f46ccfb6d8dd3ebb05bcc8 Mon Sep 17 00:00:00 2001 From: nyagami Date: Wed, 1 May 2024 00:29:53 +0700 Subject: [PATCH] enhance TTS (#1054) * fix: read by paragraph and empty string * add voice picker modal * add tts voice picker for preview * fix: tts for bionic reading * stop speak when get out of reader * search voice * temporary fix: original font * fix: start & resume * fix: close context menu on copy --- android/app/src/main/assets/css/index.css | 2 +- android/app/src/main/assets/js/index.js | 151 ++++++++---------- android/app/src/main/assets/js/type.d.ts | 5 +- src/hooks/persisted/useSettings.ts | 10 ++ src/screens/reader/ReaderScreen.tsx | 2 + .../reader/components/WebViewReader.tsx | 5 + .../Modals/VoicePickerModal.tsx | 94 +++++++++++ .../Settings/TextToSpeechSettings.tsx | 33 ++++ .../SettingsReaderScreen.tsx | 8 + 9 files changed, 223 insertions(+), 87 deletions(-) create mode 100644 src/screens/settings/SettingsReaderScreen/Modals/VoicePickerModal.tsx create mode 100644 src/screens/settings/SettingsReaderScreen/Settings/TextToSpeechSettings.tsx diff --git a/android/app/src/main/assets/css/index.css b/android/app/src/main/assets/css/index.css index 8203a47b..faa74800 100644 --- a/android/app/src/main/assets/css/index.css +++ b/android/app/src/main/assets/css/index.css @@ -307,7 +307,7 @@ td { text-align: right; } -tts.highlight { +.highlight { color: var(--theme-onSecondary); background-color: var(--theme-secondary); } diff --git a/android/app/src/main/assets/js/index.js b/android/app/src/main/assets/js/index.js index cf12deb2..73c602d9 100644 --- a/android/app/src/main/assets/js/index.js +++ b/android/app/src/main/assets/js/index.js @@ -101,14 +101,19 @@ class Reader { '--readerSettings-fontFamily', settings.fontFamily, ); - new FontFace( - settings.fontFamily, - 'url("file:///android_asset/fonts/' + settings.fontFamily + '.ttf")', - ) - .load() - .then(function (loadedFont) { - document.fonts.add(loadedFont); - }); + if (settings.fontFamily) { + new FontFace( + settings.fontFamily, + 'url("file:///android_asset/fonts/' + settings.fontFamily + '.ttf")', + ) + .load() + .then(function (loadedFont) { + document.fonts.add(loadedFont); + }); + } else { + // have no affect with a font declared in head + document.fonts.forEach(fontFace => document.fonts.delete(fontFace)); + } }; updateGeneralSettings = settings => { this.showScrollPercentage = settings.showScrollPercentage; @@ -314,12 +319,15 @@ class SwipeHandler { /** * @type {import('./type').TextToSpeech} */ + +// Walk through all leave -> convert each leaf to list of tts elements -> read through all tts elements class TextToSpeech { constructor(reader) { this.reader = reader; - (this.leaf = this.reader.chapter), (this.TTSWrapper = null); // swap these 2 elements - this.TTSEle = null; + this.readableNodeNames = ['#text', 'B', 'I', 'SPAN', 'EM', 'BR', 'STRONG']; + this.leaf = this.reader.chapter; this.reading = false; + this.started = false; this.$ = document.getElementById('TTS-Controller'); this.$.classList.add('d-none'); this.$.onclick = () => { @@ -332,9 +340,11 @@ class TextToSpeech { } start = () => { + this.started = false; this.$.innerHTML = ``; this.$.classList.remove('d-none'); this.stop(); + this.leaf = this.reader.chapter; this.next(); }; @@ -343,11 +353,11 @@ class TextToSpeech { this.$.classList.remove('d-none'); this.stop(); if (this.reader.selection.type === 'Range') { - if (this.leaf && this.TTSWrapper) { - this.TTSWrapper.replaceWith(this.leaf); - } this.leaf = this.reader.selection.anchorNode; - this.makeLeafSpeakable(); + while (this.readableNodeNames.includes(this.leaf.nodeName)) { + this.leaf = this.leaf.parentNode; + } + this.reader.selection.removeAllRanges(); this.speak(); } else { this.next(); @@ -359,77 +369,58 @@ class TextToSpeech { this.next(); }; - findLeaf() { - while (this.leaf.firstChild) { - this.leaf = this.leaf.firstChild; - } - } - - findNextLeaf() { - if (this.reader.chapter.isSameNode(this.leaf)) { - this.findLeaf(); - } else if (this.leaf.nextSibling) { - this.leaf = this.leaf.nextSibling; - this.findLeaf(); - } else { - this.leaf = this.leaf.parentNode; - if (this.reader.chapter.isSameNode(this.leaf)) { - return; - } - this.findNextLeaf(); - } - } - readable() { - return ( - this.leaf.nodeName === '#text' && /[^\s\n,.?!;":“”]/.test(this.leaf.data) - ); + if ( + this.leaf.nodeName !== 'SPAN' && + this.readableNodeNames.includes(this.leaf.nodeName) + ) { + return false; + } + if (!this.leaf.hasChildNodes()) { + return false; + } + const a = []; + for (let i = 0; i < this.leaf.childNodes.length; i++) { + if ( + !this.readableNodeNames.includes(this.leaf.childNodes.item(i).nodeName) + ) { + return false; + } + } + return true; } findNextTextNode() { - if (this.leaf && this.TTSWrapper) { - this.TTSWrapper.replaceWith(this.leaf); - this.TTSWrapper = null; - } do { - this.findNextLeaf(); - } while (!this.readable() && !this.reader.chapter.isSameNode(this.leaf)); + if (this.reader.chapter.isSameNode(this.leaf)) { + this.leaf = this.leaf.firstElementChild; + } else { + if (this.leaf.firstElementChild) { + this.leaf = this.leaf.firstElementChild; + } else if (this.leaf.nextElementSibling) { + this.leaf = this.leaf.nextElementSibling; + } else { + this.leaf = this.leaf.parentElement; + } + } + } while (!this.readable()); if (this.reader.chapter.isSameNode(this.leaf)) { return; } - this.makeLeafSpeakable(); - } - - makeLeafSpeakable() { - this.TTSWrapper = document.createElement('tts-wrapper'); - this.TTSWrapper.innerHTML = this.leaf.data.replace( - /([^\n,.?!;":“”]+)/g, - matched => { - if (matched.trim().length) { - return `${matched}`; - } - return matched; - }, - ); - this.TTSEle = this.TTSWrapper.firstElementChild; - this.leaf.replaceWith(this.TTSWrapper); } next = () => { try { - if (this.TTSEle) { - this.TTSEle.classList.remove('highlight'); - } + this.leaf?.classList?.remove('highlight'); if (this.reading) { - if (this.TTSEle && this.TTSEle.nextElementSibling) { - this.TTSEle = this.TTSEle.nextElementSibling; + if (this.leaf.nextElementSibling) { + this.leaf = this.leaf.nextElementSibling; } else { this.findNextTextNode(); } - } else { - if (!this.TTSEle) { - this.findNextTextNode(); - } + } else if (!this.started) { + this.findNextTextNode(); + this.started = true; } this.speak(); } catch (e) { @@ -438,15 +429,10 @@ class TextToSpeech { }; stop = () => { - this.$.classList.add('d-none'); - this.reading = false; - if (this.leaf && this.TTSWrapper) { - this.TTSWrapper.replaceWith(this.leaf); - this.TTSWrapper = null; - this.TTSEle = null; - this.leaf = this.reader.chapter; - } this.reader.post({ type: 'stop-speak' }); + this.$.classList.add('d-none'); + this.leaf?.classList?.remove('highlight'); + this.reading = false; }; pause = () => { @@ -455,15 +441,13 @@ class TextToSpeech { this.reader.post({ type: 'stop-speak' }); }; - started = () => this.TTSWrapper !== null; - speak = () => { this.reading = true; if (this.reader.chapter.isSameNode(this.leaf)) { return; } - this.TTSEle.classList.add('highlight'); - this.reader.post({ type: 'speak', data: this.TTSEle?.innerText.trim() }); + this.leaf.classList.add('highlight'); + this.reader.post({ type: 'speak', data: this.leaf?.innerText }); }; } @@ -481,22 +465,23 @@ class ContextMenu { name: 'Start Reading', icon: volumeIcon, action: () => { - tts.start(); this.closeMenu(); + tts.start(); }, }), START_HERE: this.renderItem({ name: 'Start Here', icon: selectVolumeIcon, action: () => { - tts.startHere(); this.closeMenu(); + tts.startHere(); }, }), COPY: this.renderItem({ name: 'Copy', icon: copyIcon, action: () => { + this.closeMenu(); this.reader.post({ type: 'copy', data: this.reader.selection.toString(), diff --git a/android/app/src/main/assets/js/type.d.ts b/android/app/src/main/assets/js/type.d.ts index 76c788f7..64365f9c 100644 --- a/android/app/src/main/assets/js/type.d.ts +++ b/android/app/src/main/assets/js/type.d.ts @@ -30,10 +30,9 @@ interface Reader { interface TextToSpeech { reader: Reader; - leaf: HTMLElement; - TTSWrapper: HTMLElement; - TTSEle: HTMLElement; + leaf: HTMLUnknownElement; reading: boolean; + readableTags: string[]; start: () => void; startHere: () => void; resume: () => void; diff --git a/src/hooks/persisted/useSettings.ts b/src/hooks/persisted/useSettings.ts index c92c9cfb..775a7443 100644 --- a/src/hooks/persisted/useSettings.ts +++ b/src/hooks/persisted/useSettings.ts @@ -4,6 +4,7 @@ import { LibrarySortOrder, } from '@screens/library/constants/constants'; import { useMMKVObject } from 'react-native-mmkv'; +import { Voice } from 'expo-speech'; export const APP_SETTINGS = 'APP_SETTINGS'; export const BROWSE_SETTINGS = 'BROWSE_SETTINGS'; @@ -100,6 +101,11 @@ export interface ChapterReaderSettings { customCSS: string; customJS: string; customThemes: ReaderTheme[]; + tts?: { + voice?: Voice; + rate?: number; + pitch?: number; + }; } const initialAppSettings: AppSettings = { @@ -173,6 +179,10 @@ export const initialChapterReaderSettings: ChapterReaderSettings = { customCSS: '', customJS: '', customThemes: [], + tts: { + rate: 1, + pitch: 1, + }, }; export const useAppSettings = () => { diff --git a/src/screens/reader/ReaderScreen.tsx b/src/screens/reader/ReaderScreen.tsx index bb3dabea..c0cd8800 100644 --- a/src/screens/reader/ReaderScreen.tsx +++ b/src/screens/reader/ReaderScreen.tsx @@ -11,6 +11,7 @@ import { DrawerLayoutAndroid, } from 'react-native'; import * as RNFS from 'react-native-fs'; +import * as Speech from 'expo-speech'; import VolumeButtonListener from '@native/volumeButtonListener'; @@ -158,6 +159,7 @@ export const ChapterContent = ({ VolumeButtonListener.disconnect(); emmiter.current.removeAllListeners('VolumeUp'); emmiter.current.removeAllListeners('VolumeDown'); + Speech.stop(); }; }, [useVolumeButtons, chapter]); diff --git a/src/screens/reader/components/WebViewReader.tsx b/src/screens/reader/components/WebViewReader.tsx index 47aa2ffd..c6ee4746 100644 --- a/src/screens/reader/components/WebViewReader.tsx +++ b/src/screens/reader/components/WebViewReader.tsx @@ -168,7 +168,12 @@ const WebViewReader: FC = props => { onDone() { webViewRef.current?.injectJavaScript('tts.next?.()'); }, + voice: readerSettings.tts?.voice?.identifier, + pitch: readerSettings.tts?.pitch || 1, + rate: readerSettings.tts?.rate || 1, }); + } else { + webViewRef.current?.injectJavaScript('tts.next?.()'); } break; case 'stop-speak': diff --git a/src/screens/settings/SettingsReaderScreen/Modals/VoicePickerModal.tsx b/src/screens/settings/SettingsReaderScreen/Modals/VoicePickerModal.tsx new file mode 100644 index 00000000..f5851409 --- /dev/null +++ b/src/screens/settings/SettingsReaderScreen/Modals/VoicePickerModal.tsx @@ -0,0 +1,94 @@ +import React, { useEffect, useState } from 'react'; +import { StyleSheet } from 'react-native'; + +import { Portal, Modal, overlay, TextInput } from 'react-native-paper'; +import { RadioButton } from '@components/RadioButton/RadioButton'; + +import { useChapterReaderSettings, useTheme } from '@hooks/persisted'; +import { Voice, getAvailableVoicesAsync } from 'expo-speech'; +import { FlashList } from '@shopify/flash-list'; + +interface VoicePickerModalProps { + visible: boolean; + onDismiss: () => void; +} + +const VoicePickerModal: React.FC = ({ + onDismiss, + visible, +}) => { + const theme = useTheme(); + const [voices, setVoices] = useState([]); + const [searchedVoices, setSearchedVoices] = useState([]); + const [searchText, setSearchText] = useState(''); + const { setChapterReaderSettings, tts } = useChapterReaderSettings(); + useEffect(() => { + getAvailableVoicesAsync().then(res => { + res.sort((a, b) => a.name.localeCompare(b.name)); + setVoices([{ name: 'System', language: 'System' } as Voice, ...res]); + }); + }, []); + return ( + + + { + setSearchText(text); + setSearchedVoices( + voices.filter(voice => + voice.name + .toLocaleLowerCase() + .includes(text.toLocaleLowerCase()), + ), + ); + }} + value={searchText} + placeholder="Search voice" + /> + } + ListHeaderComponentStyle={{ paddingHorizontal: 12 }} + data={searchText ? searchedVoices : voices} + extraData={tts?.voice} + renderItem={({ item }) => ( + + setChapterReaderSettings({ tts: { ...tts, voice: item } }) + } + label={item.name + ` (${item.language})`} + labelStyle={{ fontFamily: item.name }} + theme={theme} + /> + )} + keyExtractor={item => item.identifier || 'system'} + estimatedItemSize={64} + removeClippedSubviews={true} + /> + + + ); +}; + +export default VoicePickerModal; + +const styles = StyleSheet.create({ + containerStyle: { + paddingVertical: 24, + margin: 20, + borderRadius: 28, + flex: 1, + }, +}); diff --git a/src/screens/settings/SettingsReaderScreen/Settings/TextToSpeechSettings.tsx b/src/screens/settings/SettingsReaderScreen/Settings/TextToSpeechSettings.tsx new file mode 100644 index 00000000..9f92ac12 --- /dev/null +++ b/src/screens/settings/SettingsReaderScreen/Settings/TextToSpeechSettings.tsx @@ -0,0 +1,33 @@ +import { List } from '@components'; +import { useChapterReaderSettings, useTheme } from '@hooks/persisted'; +import React from 'react'; +import VoicePickerModal from '../Modals/VoicePickerModal'; +import { useBoolean } from '@hooks'; +import { Portal } from 'react-native-paper'; + +export default function TextToSpeechSettings() { + const theme = useTheme(); + const { tts } = useChapterReaderSettings(); + const { + value: voiceModalVisible, + setTrue: showVoiceModal, + setFalse: hideVoiceModal, + } = useBoolean(); + return ( + <> + Text to Speech + + + + + + ); +} diff --git a/src/screens/settings/SettingsReaderScreen/SettingsReaderScreen.tsx b/src/screens/settings/SettingsReaderScreen/SettingsReaderScreen.tsx index 974965b2..39da4906 100644 --- a/src/screens/settings/SettingsReaderScreen/SettingsReaderScreen.tsx +++ b/src/screens/settings/SettingsReaderScreen/SettingsReaderScreen.tsx @@ -25,6 +25,7 @@ import { useBatteryLevel } from 'react-native-device-info'; import * as Speech from 'expo-speech'; import * as Clipboard from 'expo-clipboard'; import { showToast } from '@utils/showToast'; +import TextToSpeechSettings from './Settings/TextToSpeechSettings'; export type TextAlignments = | 'left' @@ -141,7 +142,12 @@ const SettingsReaderScreen = () => { onDone() { webViewRef.current?.injectJavaScript('tts.next?.()'); }, + voice: readerSettings.tts?.voice?.identifier, + pitch: readerSettings.tts?.pitch || 1, + rate: readerSettings.tts?.rate || 1, }); + } else { + webViewRef.current?.injectJavaScript('tts.next?.()'); } break; case 'stop-speak': @@ -230,6 +236,8 @@ const SettingsReaderScreen = () => { + +