Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Small fixes and add extra test

pull/15/head
neobooru 2023-07-05 13:04:41 +02:00
parent e208d69695
commit 5beea8cc69
2 changed files with 25 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/engines/Philomena.test.ts
View File

@ -9,6 +9,22 @@ test("get derpibooru.org post 3153524", async () => {
expect(res.posts[0]).toHaveTag("yay"); expect(res.posts[0]).toHaveTag("yay");
expect(res.posts[0]).toHaveTag("flutteryay"); expect(res.posts[0]).toHaveTag("flutteryay");
expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640"); expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640");
expect(res.posts[0]).toHaveResolution([4000, 4000]);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg"); expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg");
}); });
test("get derpibooru.org post 3157356", async () => {
const pageUrl = "https://derpibooru.org/images/3157356?q=oc";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0]).not.toHaveTag("safe");
expect(res.posts[0]).toHaveTag("oc");
expect(res.posts[0]).toHaveTag("serra20", "artist");
expect(res.posts[0]).toHaveTag("pony", "species");
expect(res.posts[0]).toHaveTag("eris_(oc)", "character");
expect(res.posts[0]).toHaveResolution([960, 960]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/5/3157356.png");
});

15
src/engines/Philomena.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { SafetyRating, TagCategory } from "../BooruTypes.js"; import { SafetyRating, TagCategory } from "../BooruTypes.js";
import { parseResolutionString } from "../Utility.js";
export default class Philomena extends ScrapeEngineBase { export default class Philomena extends ScrapeEngineBase {
name = "Philomena"; name = "Philomena";
@ -15,7 +16,7 @@ export default class Philomena extends ScrapeEngineBase {
"twibooru.org", "twibooru.org",
]; ];
removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"]; removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"];
removeprefixesSuffix = new Map([["oc", "_(oc)"]]); removePrefixesSuffix = new Map([["oc", "_(oc)"]]);
artistPrefixes = ["artist", "editor"]; artistPrefixes = ["artist", "editor"];
safetyMap = new Map<string, SafetyRating>([ safetyMap = new Map<string, SafetyRating>([
["safe", "safe"], ["safe", "safe"],
@ -48,10 +49,12 @@ export default class Philomena extends ScrapeEngineBase {
post.rating = "unsafe"; post.rating = "unsafe";
// Set resolution // Set resolution
const resolutionEl = ((<HTMLElement>document.querySelector("span[class='image-size']"))?.innerText).split(" ")[0]; const resStr = (<HTMLElement>document.querySelector("span[class='image-size']"))?.innerText?.split(" ")[0];
const [width, height] = (resolutionEl?.replace("&nbsp;", "").split("x") || []).map(Number); if (resStr) {
if (!isNaN(width) && !isNaN(height)) { const res = parseResolutionString(resStr);
post.resolution = [width, height]; if (res) {
post.resolution = res;
}
} }
// Set tags // Set tags
@ -95,7 +98,7 @@ export default class Philomena extends ScrapeEngineBase {
tagName = tagName.replace(prefix + ":", "").trim(); tagName = tagName.replace(prefix + ":", "").trim();
// Add suffix if configured // Add suffix if configured
const suffix = this.removeprefixesSuffix.get(prefix); const suffix = this.removePrefixesSuffix.get(prefix);
if (suffix) { if (suffix) {
tagName = tagName + suffix; tagName = tagName + suffix;
} }