feat: fetch full articles from the Portuguese Minecraft Wiki
This commit is contained in:
marcos
@@ -0,0 +1,131 @@
|
||||
package dev.marcospaulo.canalhandia;
|
||||
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Reads articles from the Portuguese Minecraft Wiki.
|
||||
*
|
||||
* <p>Full article text, not {@code exintro}: grounded on lead paragraphs alone
|
||||
* the model answered "não tenho certeza" to questions it could otherwise get
|
||||
* right, because the specifics live further down the page.
|
||||
*
|
||||
* <p>Note that {@code explaintext} drops tables, so crafting and brewing
|
||||
* recipes never appear here. Those come from the running server's recipe
|
||||
* registry instead.
|
||||
*/
|
||||
final class Wiki {
|
||||
|
||||
private static final String API = "https://pt.minecraft.wiki/api.php";
|
||||
|
||||
record Article(String title, String text) {
|
||||
}
|
||||
|
||||
private final Fetcher fetcher;
|
||||
private final int maxChars;
|
||||
/**
|
||||
* Keyed by the normalised <em>search term</em>, not by article title: two
|
||||
* terms that resolve to the same article get their own entry, which costs a
|
||||
* duplicate copy of the text and saves a round trip on each. Newest-last,
|
||||
* evicted at 200 entries. Lost on restart, which is fine.
|
||||
*
|
||||
* <p>Guarded by its own monitor. {@link Ai} calls {@link #lookup} from
|
||||
* {@code runTaskAsynchronously}, so several players asking at once means
|
||||
* several threads in here at the same time, and an unsynchronised
|
||||
* {@code LinkedHashMap} can corrupt its own links under a concurrent write.
|
||||
* The lock is never held across a network call.
|
||||
*/
|
||||
private final Map<String, Article> cache = new LinkedHashMap<>();
|
||||
|
||||
Wiki(Fetcher fetcher, int maxChars) {
|
||||
this.fetcher = fetcher;
|
||||
this.maxChars = maxChars;
|
||||
}
|
||||
|
||||
/** The best article for a search term, or null if there is none. */
|
||||
Article lookup(String term) {
|
||||
String key = key(term);
|
||||
synchronized (cache) {
|
||||
Article cached = cache.get(key);
|
||||
if (cached != null) {
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
try {
|
||||
String title = search(term);
|
||||
if (title == null) {
|
||||
return null;
|
||||
}
|
||||
String text = extract(title);
|
||||
if (text == null || text.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
Article article = new Article(title, trim(text));
|
||||
remember(key, article);
|
||||
return article;
|
||||
} catch (Exception e) {
|
||||
// A wiki outage must not break the answer; the caller falls back
|
||||
// to answering without a source and says so.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String search(String term) throws Exception {
|
||||
String url = API + "?action=query&list=search&format=json&srlimit=1&srsearch="
|
||||
+ URLEncoder.encode(term, StandardCharsets.UTF_8);
|
||||
JsonObject root = JsonParser.parseString(fetcher.get(url)).getAsJsonObject();
|
||||
if (!root.has("query")) {
|
||||
return null;
|
||||
}
|
||||
JsonArray hits = root.getAsJsonObject("query").getAsJsonArray("search");
|
||||
return hits.isEmpty() ? null : hits.get(0).getAsJsonObject().get("title").getAsString();
|
||||
}
|
||||
|
||||
private String extract(String title) throws Exception {
|
||||
String url = API + "?action=query&prop=extracts&explaintext=1&format=json&titles="
|
||||
+ URLEncoder.encode(title, StandardCharsets.UTF_8);
|
||||
JsonObject root = JsonParser.parseString(fetcher.get(url)).getAsJsonObject();
|
||||
if (!root.has("query")) {
|
||||
return null;
|
||||
}
|
||||
JsonObject pages = root.getAsJsonObject("query").getAsJsonObject("pages");
|
||||
for (String key : pages.keySet()) {
|
||||
JsonObject page = pages.getAsJsonObject(key);
|
||||
if (page.has("extract")) {
|
||||
return page.get("extract").getAsString();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String trim(String text) {
|
||||
String collapsed = text.replaceAll("\n{2,}", "\n").trim();
|
||||
return collapsed.length() > maxChars ? collapsed.substring(0, maxChars) : collapsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@code Locale.ROOT} rather than the default locale: a server started
|
||||
* under a Turkish locale lowercases "I" to a dotless "ı", and any code that
|
||||
* later calls {@code Locale.setDefault} would leave earlier entries keyed
|
||||
* under rules no lookup uses again.
|
||||
*/
|
||||
private static String key(String term) {
|
||||
return term.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private void remember(String key, Article article) {
|
||||
synchronized (cache) {
|
||||
cache.put(key, article);
|
||||
while (cache.size() > 200) {
|
||||
cache.remove(cache.keySet().iterator().next());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package dev.marcospaulo.canalhandia;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import java.util.Map;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class WikiTest {
|
||||
|
||||
/** Returns canned bodies keyed by a substring of the requested URL. */
|
||||
private static Fetcher stub(Map<String, String> byUrlFragment) {
|
||||
return new Fetcher() {
|
||||
@Override
|
||||
public String get(String url) {
|
||||
for (Map.Entry<String, String> e : byUrlFragment.entrySet()) {
|
||||
if (url.contains(e.getKey())) {
|
||||
return e.getValue();
|
||||
}
|
||||
}
|
||||
throw new AssertionError("unexpected url: " + url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String postJson(String url, String json, String bearer) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Test
|
||||
void findsArticleAndReturnsFullText() {
|
||||
Wiki wiki = new Wiki(stub(Map.of(
|
||||
"list=search", "{\"query\":{\"search\":[{\"title\":\"Camelo\"}]}}",
|
||||
"prop=extracts", "{\"query\":{\"pages\":{\"1\":{\"title\":\"Camelo\","
|
||||
+ "\"extract\":\"Um camelo pode ser equipado com uma sela.\"}}}}")),
|
||||
7000);
|
||||
|
||||
Wiki.Article article = wiki.lookup("Camelo");
|
||||
|
||||
assertNotNull(article);
|
||||
assertEquals("Camelo", article.title());
|
||||
assertTrue(article.text().contains("sela"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void returnsNullWhenNothingMatches() {
|
||||
Wiki wiki = new Wiki(stub(Map.of(
|
||||
"list=search", "{\"query\":{\"search\":[]}}")), 7000);
|
||||
assertNull(wiki.lookup("asdfghjkl"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void truncatesLongArticles() {
|
||||
String longText = "x".repeat(9000);
|
||||
Wiki wiki = new Wiki(stub(Map.of(
|
||||
"list=search", "{\"query\":{\"search\":[{\"title\":\"T\"}]}}",
|
||||
"prop=extracts", "{\"query\":{\"pages\":{\"1\":{\"title\":\"T\","
|
||||
+ "\"extract\":\"" + longText + "\"}}}}")), 100);
|
||||
assertEquals(100, wiki.lookup("T").text().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
void cachesByTitleSoRepeatQuestionsCostOneFetch() {
|
||||
int[] calls = {0};
|
||||
Fetcher counting = new Fetcher() {
|
||||
@Override
|
||||
public String get(String url) {
|
||||
calls[0]++;
|
||||
return url.contains("list=search")
|
||||
? "{\"query\":{\"search\":[{\"title\":\"Creeper\"}]}}"
|
||||
: "{\"query\":{\"pages\":{\"1\":{\"title\":\"Creeper\",\"extract\":\"polvora\"}}}}";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String postJson(String url, String json, String bearer) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
};
|
||||
Wiki wiki = new Wiki(counting, 7000);
|
||||
wiki.lookup("Creeper");
|
||||
int afterFirst = calls[0];
|
||||
wiki.lookup("Creeper");
|
||||
assertEquals(afterFirst, calls[0], "second lookup should be served from cache");
|
||||
}
|
||||
|
||||
@Test
|
||||
void networkFailureYieldsNullRatherThanThrowing() {
|
||||
Wiki wiki = new Wiki(new Fetcher() {
|
||||
@Override
|
||||
public String get(String url) throws java.io.IOException {
|
||||
throw new java.io.IOException("boom");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String postJson(String url, String json, String bearer) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}, 7000);
|
||||
assertNull(wiki.lookup("Camelo"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user