From 58f6475749c3be0c19e32d8bde4f2acbe31118e6 Mon Sep 17 00:00:00 2001 From: nyagami Date: Sun, 28 Jan 2024 18:43:20 +0700 Subject: [PATCH] update: use cheerio to parse local novel --- .../EpubParser/EpubParser.java | 174 ------------------ .../EpubParser/ZipArchive.java | 58 ++++++ ...serPackage.java => ZipArchivePackage.java} | 4 +- .../LNReader/MainApplication.java | 4 +- src/native/ZipArchive.ts | 8 + src/native/epubParser.ts | 8 - src/services/epub/import.ts | 160 +++++++++++----- 7 files changed, 188 insertions(+), 228 deletions(-) delete mode 100644 android/app/src/main/java/com/rajarsheechatterjee/EpubParser/EpubParser.java create mode 100644 android/app/src/main/java/com/rajarsheechatterjee/EpubParser/ZipArchive.java rename android/app/src/main/java/com/rajarsheechatterjee/EpubParser/{EpubParserPackage.java => ZipArchivePackage.java} (87%) create mode 100644 src/native/ZipArchive.ts delete mode 100644 src/native/epubParser.ts diff --git a/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/EpubParser.java b/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/EpubParser.java deleted file mode 100644 index b217df89..00000000 --- a/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/EpubParser.java +++ /dev/null @@ -1,174 +0,0 @@ -package com.rajarsheechatterjee.EpubParser; - -import androidx.annotation.NonNull; - -import com.facebook.react.bridge.Promise; -import com.facebook.react.bridge.ReactApplicationContext; -import com.facebook.react.bridge.ReactContextBaseJavaModule; -import com.facebook.react.bridge.ReactMethod; -import com.facebook.react.bridge.ReadableArray; -import com.facebook.react.bridge.ReadableMap; -import com.facebook.react.bridge.WritableArray; -import com.facebook.react.bridge.WritableMap; -import com.facebook.react.bridge.WritableNativeArray; -import com.facebook.react.bridge.WritableNativeMap; - -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.util.regex.Pattern; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathFactory; - -public class EpubParser extends ReactContextBaseJavaModule { - String rootPath; - String contentDirPath; - XPath xPath = XPathFactory.newInstance().newXPath(); - DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - - EpubParser(ReactApplicationContext context) throws Exception { - super(context); - } - - @NonNull - @Override - public String getName() { - return "EpubParser"; - } - - @ReactMethod - public void parse(String epubFilePath, String epubDirPath, Promise promise) { - try{ - this.rootPath = epubDirPath; - String novelCover = this.unzip(epubFilePath, epubDirPath); - File contentFile = this.parseContainer(epubDirPath); - this.contentDirPath = contentFile.getParent(); - WritableMap novel = (WritableMap) this.parseContent(contentFile); - if(novelCover != null){ - novel.putString("cover", new File(epubDirPath, novelCover).getAbsolutePath()); - } - promise.resolve(novel); - } catch (Exception e) { - promise.reject(e.getCause()); - } - } - - public File parseContainer(String epubDir) throws Exception { - Document doc = this.parserDocument(new File(epubDir, "META-INF/container.xml")); - return new File(this.rootPath, - ((Element) doc.getElementsByTagName("rootfile").item(0)).getAttribute("full-path")); - } - - public ReadableMap parseContent(File contentFile) throws Exception { - Document content = this.parserDocument(contentFile); - WritableMap novel = new WritableNativeMap(); - novel.putString("name", this.getFirstElementByTag(content, "dc:title").getTextContent().trim()); - novel.putString("url", contentFile.getParent()); - novel.putString("author", this.getFirstElementByTag(content, "dc:creator").getTextContent().trim()); - novel.putArray("chapters", this.getChapters(content)); - return novel; - } - - public ReadableArray getChapters(Document doc) throws Exception { - WritableArray chapters = new WritableNativeArray(); - Document toc = null; - if (this.getFirstElementByTag(doc, "spine").getAttribute("toc") != null) { - toc = this.parserDocument(new File(this.contentDirPath, "toc.ncx")); - } - NodeList itemrefs = this.getNodeListByTag(doc, "itemref"); - for (int i = 0; i < itemrefs.getLength(); i++) { - WritableMap chapter = new WritableNativeMap(); - Element itemref = (Element) itemrefs.item(i); - chapter.putString("name", itemref.getAttribute("idref")); - Element item = this.getElementById(doc, chapter.getString("name")); - chapter.putString("url", new File(this.contentDirPath, item.getAttribute("href")).getAbsolutePath()); - if(toc != null){ - chapter.putString("name", this.getChapterName(toc, item.getAttribute(("href")))); - if(chapter.getString("name") == null) continue; - } - chapters.pushMap(chapter); - } - return chapters; - } - - public String getChapterName(Document doc, String href) throws Exception { - String path = String.format("//*[starts-with(@src,'%1$s')]", href); - Node node = ((NodeList) this.xPath.compile(path).evaluate(doc, XPathConstants.NODESET)).item(0); - if(node != null){ - String text = node.getPreviousSibling().getTextContent().trim(); - if(text.isEmpty()) text = node.getPreviousSibling().getPreviousSibling().getTextContent().trim(); - return !text.isEmpty() ? text : null; - } - return null; - } - - public Element getElementById(Document doc, String id) throws Exception { - return (Element) ((NodeList) this.xPath - .compile(String.format("//*[@id='%1$s']", id)) - .evaluate(doc, XPathConstants.NODESET)) - .item(0); - } - - public NodeList getNodeListByTag(Document doc, String tag) throws Exception { - return (NodeList) - ( - this.xPath.compile(String.format("//*[contains(name(), '%1$s')]", tag)) - .evaluate(doc, XPathConstants.NODESET) - ); - } - - public Element getFirstElementByTag(Document doc, String tag) throws Exception { - return (Element) this.getNodeListByTag(doc, tag).item(0); - } - - public Document parserDocument(File docFile) throws Exception { - Document doc = this.db.parse(docFile); - doc.getDocumentElement().normalize(); - return doc; - } - - public String escapeFilePath(String filePath){ - return filePath.replaceAll(":", "\uA789"); - } - - public String unzip(String epubFilePath, String epubDirPath) throws Exception { - ZipInputStream zis = new ZipInputStream(new FileInputStream(epubFilePath)); - ZipEntry zipEntry; - int len; - String novelCover = null; - byte[] buffer = new byte[4096]; - while ((zipEntry = zis.getNextEntry()) != null) { - String escapedFilePath = escapeFilePath(zipEntry.getName()); - if(Pattern.compile(".*cover.*\\.(png|jpeg|jpg)", Pattern.CASE_INSENSITIVE).matcher(escapedFilePath).matches()){ - novelCover = escapedFilePath; - } - File newFile = new File(epubDirPath, escapedFilePath); - if(newFile.exists()) continue; - newFile.getParentFile().mkdirs(); - FileOutputStream fos = new FileOutputStream(newFile); - int writeCount = 0; - while ((len = zis.read(buffer)) > 0) { - writeCount += 1; - fos.write(buffer, 0, len); - } - fos.close(); - if(writeCount == 0){ - newFile.delete(); - } - } - zis.closeEntry(); - zis.close(); - return novelCover; - } -} diff --git a/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/ZipArchive.java b/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/ZipArchive.java new file mode 100644 index 00000000..19ddac9c --- /dev/null +++ b/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/ZipArchive.java @@ -0,0 +1,58 @@ +package com.rajarsheechatterjee.EpubParser; + +import androidx.annotation.NonNull; + +import com.facebook.react.bridge.Promise; +import com.facebook.react.bridge.ReactApplicationContext; +import com.facebook.react.bridge.ReactContextBaseJavaModule; +import com.facebook.react.bridge.ReactMethod; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class ZipArchive extends ReactContextBaseJavaModule { + ZipArchive(ReactApplicationContext context) throws Exception { + super(context); + } + + @NonNull + @Override + public String getName() { + return "ZipArchive"; + } + public String escapeFilePath(String filePath){ + return filePath.replaceAll(":", "\uA789"); + } + + @ReactMethod + public void unzip(String epubFilePath, String epubDirPath, Promise promise) { + try{ + ZipInputStream zis = new ZipInputStream(new FileInputStream(epubFilePath)); + ZipEntry zipEntry; + int len; + byte[] buffer = new byte[4096]; + while ((zipEntry = zis.getNextEntry()) != null) { + String escapedFilePath = this.escapeFilePath(zipEntry.getName()); + File newFile = new File(epubDirPath, escapedFilePath); + newFile.getParentFile().mkdirs(); + FileOutputStream fos = new FileOutputStream(newFile); + boolean isWritten = false; // ignore folder entry; + while ((len = zis.read(buffer)) > 0) { + isWritten = true; + fos.write(buffer, 0, len); + } + fos.close(); + if(!isWritten) newFile.delete(); + } + zis.closeEntry(); + zis.close(); + promise.resolve(null); + } catch (Exception e) { + promise.reject(e.getCause()); + } + } + +} diff --git a/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/EpubParserPackage.java b/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/ZipArchivePackage.java similarity index 87% rename from android/app/src/main/java/com/rajarsheechatterjee/EpubParser/EpubParserPackage.java rename to android/app/src/main/java/com/rajarsheechatterjee/EpubParser/ZipArchivePackage.java index 8bc80e3e..b22b4f90 100644 --- a/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/EpubParserPackage.java +++ b/android/app/src/main/java/com/rajarsheechatterjee/EpubParser/ZipArchivePackage.java @@ -11,13 +11,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -public class EpubParserPackage implements ReactPackage { +public class ZipArchivePackage implements ReactPackage { @NonNull @Override public List createNativeModules(@NonNull ReactApplicationContext reactApplicationContext) { List modules = new ArrayList<>(); try { - modules.add(new EpubParser(reactApplicationContext)); + modules.add(new ZipArchive(reactApplicationContext)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/android/app/src/main/java/com/rajarsheechatterjee/LNReader/MainApplication.java b/android/app/src/main/java/com/rajarsheechatterjee/LNReader/MainApplication.java index ac3ea302..8fe3dcc9 100644 --- a/android/app/src/main/java/com/rajarsheechatterjee/LNReader/MainApplication.java +++ b/android/app/src/main/java/com/rajarsheechatterjee/LNReader/MainApplication.java @@ -14,7 +14,7 @@ import com.facebook.soloader.SoLoader; import com.rajarsheechatterjee.NavigationBarColor.NavigationBarColorPackage; import com.rajarsheechatterjee.VolumeButtonListener.VolumeButtonListenerPackage; -import com.rajarsheechatterjee.EpubParser.EpubParserPackage; +import com.rajarsheechatterjee.EpubParser.ZipArchivePackage; import expo.modules.ApplicationLifecycleDispatcher; import expo.modules.ReactNativeHostWrapper; @@ -35,7 +35,7 @@ public class MainApplication extends Application implements ReactApplication { List packages = new PackageList(this).getPackages(); packages.add(new NavigationBarColorPackage()); packages.add(new VolumeButtonListenerPackage()); - packages.add(new EpubParserPackage()); + packages.add(new ZipArchivePackage()); return packages; } diff --git a/src/native/ZipArchive.ts b/src/native/ZipArchive.ts new file mode 100644 index 00000000..9e4de621 --- /dev/null +++ b/src/native/ZipArchive.ts @@ -0,0 +1,8 @@ +import { SourceNovel } from '@plugins/types'; +import { NativeModules } from 'react-native'; +interface ZipArchiveInterface { + unzip: (epubFilePath: string, epubDirPath: string) => Promise; +} +const { ZipArchive } = NativeModules; + +export default ZipArchive as ZipArchiveInterface; diff --git a/src/native/epubParser.ts b/src/native/epubParser.ts deleted file mode 100644 index ee416d1a..00000000 --- a/src/native/epubParser.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { SourceNovel } from '@plugins/types'; -import { NativeModules } from 'react-native'; -interface EpubParserInterface { - parse: (epubFilePath: string, epubDirPath: string) => Promise; -} -const { EpubParser } = NativeModules; - -export default EpubParser as EpubParserInterface; diff --git a/src/services/epub/import.ts b/src/services/epub/import.ts index 667f0eb5..40725341 100644 --- a/src/services/epub/import.ts +++ b/src/services/epub/import.ts @@ -1,12 +1,11 @@ import * as SQLite from 'expo-sqlite'; const db = SQLite.openDatabase('lnreader.db'); -import { SourceNovel } from '@plugins/types'; import * as DocumentPicker from 'expo-document-picker'; import * as RNFS from 'react-native-fs'; import BackgroundService from 'react-native-background-actions'; import * as Notifications from 'expo-notifications'; import { sleep } from '@utils/sleep'; -import EpubParser from '@native/epubParser'; +import ZipArchive from '@native/ZipArchive'; import { NovelDownloadFolder } from '@utils/constants/download'; import dayjs from 'dayjs'; import { @@ -16,11 +15,12 @@ import { import { LOCAL_PLUGIN_ID } from '@plugins/pluginManager'; import { MMKVStorage } from '@utils/mmkv/mmkv'; import { BACKGROUND_ACTION, BackgoundAction } from '@services/constants'; +import { ChapterItem, SourceNovel } from '@plugins/types'; +import { load as parseXML } from 'cheerio'; interface TaskData { delay: number; - epubFilePath: string; - epubDirPath: string; // export dir + sourceUri: string; } const insertLocalNovel = ( @@ -33,7 +33,7 @@ const insertLocalNovel = ( db.transaction(tx => { tx.executeSql( "INSERT INTO Novel(name, url, cover, author, pluginId, inLibrary, isLocal) VALUES(?, ?, ?, ?, 'local', 1, 1)", - [name, url, cover || '', author || 'unknown'], + [name, url, cover || null, author || null], async (txObj, resultSet) => { if (resultSet.insertId) { await updateNovelCategoryById(resultSet.insertId, [2]); @@ -48,6 +48,7 @@ const insertLocalNovel = ( await updateNovelInfo({ pluginId: LOCAL_PLUGIN_ID, id: resultSet.insertId, + author: author, url: NovelDownloadFolder + '/local/' + resultSet.insertId, cover: newCoverPath, name: name, @@ -81,7 +82,7 @@ const insertLocalChapter = ( 'INSERT INTO Chapter(novelId, name, url, releaseTime) VALUES(?, ?, ?, ?)', [ novelId, - decodeURI(name), + name, // use fakeid just for make the url is unique :D NovelDownloadFolder + '/local/' + novelId + '/' + fakeId, releaseTime, @@ -95,20 +96,12 @@ const insertLocalChapter = ( chapterText = chapterText.replace( /(href|src)=(["'])(.*?)\2/g, ($0, $1, $2, $3: string) => { - const escapedFilePath = $3.replace(/:/g, '\uA789'); if ($3) { - staticPaths.push(epubContentDir + '/' + escapedFilePath); + staticPaths.push(epubContentDir + '/' + $3); } - return ( - $1 + - '=' + - $2 + - 'file://' + - novelDir + - '/' + - escapedFilePath.split(/[\\\/]/)?.pop() + - $2 - ); + return `${$1}="file://${novelDir}/${$3 + .split(/[\\\/]/) + ?.pop()}"`; }, ); await RNFS.mkdir(novelDir + '/' + resultSet.insertId); @@ -129,29 +122,122 @@ const insertLocalChapter = ( }); }); }; -const importEpubAction = async (taskData?: TaskData) => { - if (!taskData) { - throw new Error('No data provided'); + +const getParent = (filePath: string) => { + return filePath.replace(/[/\\][^/\\]+$/, ''); +}; + +const parseNovelAndChapters = async ( + epubDirPath: string, +): Promise => { + const containerText = await RNFS.readFile( + `${epubDirPath}/META-INF/container.xml`, + ); + const contentPath = + epubDirPath + '/' + parseXML(containerText)('rootfile').attr('full-path'); + const contentDir = getParent(contentPath); + const contentText = await RNFS.readFile(contentPath); + const parsedContent = parseXML(contentText); + const novelName = parsedContent('dc\\:title').text().trim(); + const author = parsedContent('dc\\:creator').text().trim(); + const coverRef = parsedContent('meta[name="cover"]').attr('content'); + let cover = ''; + if (coverRef) { + cover = + contentDir + '/' + parsedContent(`item[id="${coverRef}"]`).attr('href'); } - await sleep(taskData.delay); + BackgroundService.updateNotification({ + progressBar: { + value: 1, + max: 4, + }, + }); + const tocPath = contentDir + '/' + 'toc.ncx'; + const tocMap: Record = {}; + if (await RNFS.exists(tocPath)) { + const tocText = await RNFS.readFile(tocPath); + const toc = parseXML(tocText); + toc('navPoint').each(function () { + const href = toc(this).find('content').attr('src'); + const name = toc(this).find('text').text().trim(); + if (href && name) { + tocMap[href] = name; + } + }); + } + BackgroundService.updateNotification({ + progressBar: { + value: 2, + max: 4, + }, + }); + const itemMap: Record = {}; + parsedContent('item').each(function () { + itemMap[this.attribs.id] = this.attribs.href; + }); + BackgroundService.updateNotification({ + progressBar: { + value: 3, + max: 4, + }, + }); + const chapters: ChapterItem[] = parsedContent('itemref') + .toArray() + .map(ele => { + const href = itemMap[ele.attribs.idref]; + return { + name: tocMap[href], + url: `${contentDir}/${href}`, + }; + }); + BackgroundService.updateNotification({ + progressBar: { + value: 4, + max: 4, + indeterminate: true, + }, + }); + const novel: SourceNovel = { + name: novelName, + author: author, + cover: cover, + url: contentDir + novelName, // temporary + chapters: chapters, + }; + return novel; +}; + +const importEpubAction = async (taskData?: TaskData) => { try { + if (!taskData) { + throw new Error('No data provided'); + } + const epubFilePath = RNFS.ExternalCachesDirectoryPath + '/novel.epub'; + await RNFS.copyFile(taskData.sourceUri, epubFilePath); + const epubDirPath = RNFS.ExternalCachesDirectoryPath + '/epub'; + if (await RNFS.exists(epubDirPath)) { + await RNFS.unlink(epubDirPath); + } + await RNFS.mkdir(epubDirPath).catch(e => { + throw e; + }); MMKVStorage.set(BACKGROUND_ACTION, BackgoundAction.IMPORT_EPUB); - const novel: SourceNovel = await EpubParser.parse( - taskData.epubFilePath, - taskData.epubDirPath, - ).catch(e => { + await ZipArchive.unzip(epubFilePath, epubDirPath).catch(e => { throw e; }); await sleep(taskData.delay); + const novel = await parseNovelAndChapters(epubDirPath).catch(e => { + throw e; + }); const novelId = await insertLocalNovel( novel.name, novel.url, novel.cover, - novel.artist, + novel.author, ).catch(e => { throw e; }); - const now = dayjs().format('L'); + const now = dayjs().toISOString(); const filePathSet = new Set(); if (novel.chapters) { BackgroundService.updateNotification({ @@ -171,7 +257,7 @@ const importEpubAction = async (taskData?: TaskData) => { }, }); const chapter = novel.chapters[i]; - if (chapter.name == null) { + if (!chapter.name) { chapter.name = chapter.url.split(/[\\\/]/).pop() || 'unknown'; } const filePaths = await insertLocalChapter( @@ -239,29 +325,19 @@ export const importEpub = async () => { try { const epubFile = await DocumentPicker.getDocumentAsync({ type: 'application/epub+zip', - copyToCacheDirectory: true, + copyToCacheDirectory: false, }); if (epubFile.type === 'cancel') { throw new Error('Cancel'); } - const epubFilePath = RNFS.ExternalCachesDirectoryPath + '/novel.epub'; - await RNFS.moveFile(epubFile.uri, epubFilePath).catch(e => { - throw e; - }); - const epubDirPath = RNFS.ExternalCachesDirectoryPath + '/epub'; - if (await RNFS.exists(epubDirPath)) { - await RNFS.unlink(epubDirPath); - } - await RNFS.mkdir(epubDirPath).catch(e => { - throw e; - }); await BackgroundService.start(importEpubAction, { taskName: 'Import Epub', taskTitle: 'Parse Epub', taskDesc: 'Parsing', taskIcon: { name: 'notification_icon', type: 'drawable' }, color: '#00adb5', - parameters: { delay: 500, epubFilePath, epubDirPath }, + parameters: { delay: 500, sourceUri: epubFile.uri }, + progressBar: { value: 0, max: 4 }, }); } catch (e: any) { // BackgroundService.catch cant catch Error from importEpubAction