From 13fb87fb8bc71a4a6de1eb2e77825bd9009831f2 Mon Sep 17 00:00:00 2001 From: marcos Date: Wed, 5 Aug 2026 15:17:01 +0000 Subject: [PATCH] feat: fetch full articles from the Portuguese Minecraft Wiki --- .../dev/marcospaulo/canalhandia/Wiki.java | 131 ++++++++++++++++++ .../dev/marcospaulo/canalhandia/WikiTest.java | 100 +++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 src/main/java/dev/marcospaulo/canalhandia/Wiki.java create mode 100644 src/test/java/dev/marcospaulo/canalhandia/WikiTest.java diff --git a/src/main/java/dev/marcospaulo/canalhandia/Wiki.java b/src/main/java/dev/marcospaulo/canalhandia/Wiki.java new file mode 100644 index 0000000..748634f --- /dev/null +++ b/src/main/java/dev/marcospaulo/canalhandia/Wiki.java @@ -0,0 +1,131 @@ +package dev.marcospaulo.canalhandia; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +/** + * Reads articles from the Portuguese Minecraft Wiki. + * + *

Full article text, not {@code exintro}: grounded on lead paragraphs alone + * the model answered "não tenho certeza" to questions it could otherwise get + * right, because the specifics live further down the page. + * + *

Note that {@code explaintext} drops tables, so crafting and brewing + * recipes never appear here. Those come from the running server's recipe + * registry instead. + */ +final class Wiki { + + private static final String API = "https://pt.minecraft.wiki/api.php"; + + record Article(String title, String text) { + } + + private final Fetcher fetcher; + private final int maxChars; + /** + * Keyed by the normalised search term, not by article title: two + * terms that resolve to the same article get their own entry, which costs a + * duplicate copy of the text and saves a round trip on each. Newest-last, + * evicted at 200 entries. Lost on restart, which is fine. + * + *

Guarded by its own monitor. {@link Ai} calls {@link #lookup} from + * {@code runTaskAsynchronously}, so several players asking at once means + * several threads in here at the same time, and an unsynchronised + * {@code LinkedHashMap} can corrupt its own links under a concurrent write. + * The lock is never held across a network call. + */ + private final Map cache = new LinkedHashMap<>(); + + Wiki(Fetcher fetcher, int maxChars) { + this.fetcher = fetcher; + this.maxChars = maxChars; + } + + /** The best article for a search term, or null if there is none. */ + Article lookup(String term) { + String key = key(term); + synchronized (cache) { + Article cached = cache.get(key); + if (cached != null) { + return cached; + } + } + try { + String title = search(term); + if (title == null) { + return null; + } + String text = extract(title); + if (text == null || text.isBlank()) { + return null; + } + Article article = new Article(title, trim(text)); + remember(key, article); + return article; + } catch (Exception e) { + // A wiki outage must not break the answer; the caller falls back + // to answering without a source and says so. + return null; + } + } + + private String search(String term) throws Exception { + String url = API + "?action=query&list=search&format=json&srlimit=1&srsearch=" + + URLEncoder.encode(term, StandardCharsets.UTF_8); + JsonObject root = JsonParser.parseString(fetcher.get(url)).getAsJsonObject(); + if (!root.has("query")) { + return null; + } + JsonArray hits = root.getAsJsonObject("query").getAsJsonArray("search"); + return hits.isEmpty() ? null : hits.get(0).getAsJsonObject().get("title").getAsString(); + } + + private String extract(String title) throws Exception { + String url = API + "?action=query&prop=extracts&explaintext=1&format=json&titles=" + + URLEncoder.encode(title, StandardCharsets.UTF_8); + JsonObject root = JsonParser.parseString(fetcher.get(url)).getAsJsonObject(); + if (!root.has("query")) { + return null; + } + JsonObject pages = root.getAsJsonObject("query").getAsJsonObject("pages"); + for (String key : pages.keySet()) { + JsonObject page = pages.getAsJsonObject(key); + if (page.has("extract")) { + return page.get("extract").getAsString(); + } + } + return null; + } + + private String trim(String text) { + String collapsed = text.replaceAll("\n{2,}", "\n").trim(); + return collapsed.length() > maxChars ? collapsed.substring(0, maxChars) : collapsed; + } + + /** + * {@code Locale.ROOT} rather than the default locale: a server started + * under a Turkish locale lowercases "I" to a dotless "ı", and any code that + * later calls {@code Locale.setDefault} would leave earlier entries keyed + * under rules no lookup uses again. + */ + private static String key(String term) { + return term.toLowerCase(Locale.ROOT); + } + + private void remember(String key, Article article) { + synchronized (cache) { + cache.put(key, article); + while (cache.size() > 200) { + cache.remove(cache.keySet().iterator().next()); + } + } + } +} diff --git a/src/test/java/dev/marcospaulo/canalhandia/WikiTest.java b/src/test/java/dev/marcospaulo/canalhandia/WikiTest.java new file mode 100644 index 0000000..048b5bf --- /dev/null +++ b/src/test/java/dev/marcospaulo/canalhandia/WikiTest.java @@ -0,0 +1,100 @@ +package dev.marcospaulo.canalhandia; + +import org.junit.jupiter.api.Test; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.*; + +class WikiTest { + + /** Returns canned bodies keyed by a substring of the requested URL. */ + private static Fetcher stub(Map byUrlFragment) { + return new Fetcher() { + @Override + public String get(String url) { + for (Map.Entry e : byUrlFragment.entrySet()) { + if (url.contains(e.getKey())) { + return e.getValue(); + } + } + throw new AssertionError("unexpected url: " + url); + } + + @Override + public String postJson(String url, String json, String bearer) { + throw new UnsupportedOperationException(); + } + }; + } + + @Test + void findsArticleAndReturnsFullText() { + Wiki wiki = new Wiki(stub(Map.of( + "list=search", "{\"query\":{\"search\":[{\"title\":\"Camelo\"}]}}", + "prop=extracts", "{\"query\":{\"pages\":{\"1\":{\"title\":\"Camelo\"," + + "\"extract\":\"Um camelo pode ser equipado com uma sela.\"}}}}")), + 7000); + + Wiki.Article article = wiki.lookup("Camelo"); + + assertNotNull(article); + assertEquals("Camelo", article.title()); + assertTrue(article.text().contains("sela")); + } + + @Test + void returnsNullWhenNothingMatches() { + Wiki wiki = new Wiki(stub(Map.of( + "list=search", "{\"query\":{\"search\":[]}}")), 7000); + assertNull(wiki.lookup("asdfghjkl")); + } + + @Test + void truncatesLongArticles() { + String longText = "x".repeat(9000); + Wiki wiki = new Wiki(stub(Map.of( + "list=search", "{\"query\":{\"search\":[{\"title\":\"T\"}]}}", + "prop=extracts", "{\"query\":{\"pages\":{\"1\":{\"title\":\"T\"," + + "\"extract\":\"" + longText + "\"}}}}")), 100); + assertEquals(100, wiki.lookup("T").text().length()); + } + + @Test + void cachesByTitleSoRepeatQuestionsCostOneFetch() { + int[] calls = {0}; + Fetcher counting = new Fetcher() { + @Override + public String get(String url) { + calls[0]++; + return url.contains("list=search") + ? "{\"query\":{\"search\":[{\"title\":\"Creeper\"}]}}" + : "{\"query\":{\"pages\":{\"1\":{\"title\":\"Creeper\",\"extract\":\"polvora\"}}}}"; + } + + @Override + public String postJson(String url, String json, String bearer) { + throw new UnsupportedOperationException(); + } + }; + Wiki wiki = new Wiki(counting, 7000); + wiki.lookup("Creeper"); + int afterFirst = calls[0]; + wiki.lookup("Creeper"); + assertEquals(afterFirst, calls[0], "second lookup should be served from cache"); + } + + @Test + void networkFailureYieldsNullRatherThanThrowing() { + Wiki wiki = new Wiki(new Fetcher() { + @Override + public String get(String url) throws java.io.IOException { + throw new java.io.IOException("boom"); + } + + @Override + public String postJson(String url, String json, String bearer) { + throw new UnsupportedOperationException(); + } + }, 7000); + assertNull(wiki.lookup("Camelo")); + } +}