diff --git a/README.md b/README.md index 8ef8009..72f25a8 100644 --- a/README.md +++ b/README.md @@ -4,21 +4,22 @@ Scrape a booru-like post object from the DOM. ## Supported sites -| Engine | Sites | Features | Notes | -| -------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | -| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | -| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | -| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | -| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | -| inkbunny | inkbunny.net | content, rating, resolution, tags | | -| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | -| pixiv | pixiv.net, www.pixiv.net | content | | -| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | -| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | -| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | -| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | -| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | -| twitter | twitter.com, mobile.twitter.com | content | | -| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| fallback | | content | Tries to find the largest image or video on the current page. | +| Engine | Sites | Features | Notes | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | +| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | +| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | +| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | +| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | +| inkbunny | inkbunny.net | content, rating, resolution, tags | | +| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | +| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | | +| pixiv | pixiv.net, www.pixiv.net | content | | +| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | +| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | +| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | +| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | +| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | +| twitter | twitter.com, mobile.twitter.com | content | | +| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| fallback | | content | Tries to find the largest image or video on the current page. | \ No newline at end of file diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 16dd8ce..b07bfb1 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -16,6 +16,7 @@ export default class NeoScraper { new engines.Gelbooru(), new engines.Inkbunny(), new engines.Moebooru(), + new engines.Philomena(), new engines.Pixiv(), new engines.Reddit(), new engines.Rule34us(), diff --git a/src/engines/Philomena.test.ts b/src/engines/Philomena.test.ts new file mode 100644 index 0000000..155a6e9 --- /dev/null +++ b/src/engines/Philomena.test.ts @@ -0,0 +1,30 @@ +import { scrapeUrl } from "../test/engine.js"; + +test("get derpibooru.org post 3153524", async () => { + const pageUrl = "https://derpibooru.org/images/3153524"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].rating).toBe("safe"); + expect(res.posts[0]).not.toHaveTag("safe"); + expect(res.posts[0]).toHaveTag("yay"); + expect(res.posts[0]).toHaveTag("flutteryay"); + expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640"); + expect(res.posts[0]).toHaveResolution([4000, 4000]); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg"); +}); + +test("get derpibooru.org post 3157356", async () => { + const pageUrl = "https://derpibooru.org/images/3157356?q=oc"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].rating).toBe("safe"); + expect(res.posts[0]).not.toHaveTag("safe"); + expect(res.posts[0]).toHaveTag("oc"); + expect(res.posts[0]).toHaveTag("serra20", "artist"); + expect(res.posts[0]).toHaveTag("pony", "species"); + expect(res.posts[0]).toHaveTag("eris_(oc)", "character"); + expect(res.posts[0]).toHaveResolution([960, 960]); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/5/3157356.png"); +}); diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts new file mode 100644 index 0000000..a8d5c87 --- /dev/null +++ b/src/engines/Philomena.ts @@ -0,0 +1,125 @@ +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; +import { SafetyRating, TagCategory } from "../BooruTypes.js"; +import { parseResolutionString } from "../Utility.js"; + +export default class Philomena extends ScrapeEngineBase { + name = "Philomena"; + features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"]; + notes = []; + supportedHosts = [ + "derpibooru.org", + "trixiebooru.org", + "ponybooru.org", + "furbooru.org", + "ponerpics.org", + "manebooru.art", + "twibooru.org", + ]; + removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"]; + removePrefixesSuffix = new Map([["oc", "_(oc)"]]); + artistPrefixes = ["artist", "editor"]; + safetyMap = new Map([ + ["safe", "safe"], + ["suggestive", "sketchy"], + ["questionable", "sketchy"], + ["explicit", "unsafe"], + ]); + categoryMap = new Map([ + ["character", "character"], + ["species", "species"], + ["oc", "character"], + ["content-official", "copyright"], + ]); + + scrapeDocument(document: Document): ScrapeResult { + const result = new ScrapeResult(this.name); + const post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + const downloadEl = document.querySelector("a[title='Download (no tags in filename)']") as HTMLAnchorElement; + post.contentUrl = downloadEl?.href; + + // Set content type + if (document.querySelector("video#image-display") != null) { + post.contentType = "video"; + } + + // Set default rating (Will be determined later) + post.rating = "unsafe"; + + // Set resolution + const resStr = (document.querySelector("span[class='image-size']"))?.innerText?.split(" ")[0]; + if (resStr) { + const res = parseResolutionString(resStr); + if (res) { + post.resolution = res; + } + } + + // Set tags + const tagEls = Array.from(document.querySelectorAll(".tag.dropdown")).map((x) => x as HTMLSpanElement); + + for (const el of tagEls) { + let tagName = el.getAttribute("data-tag-name"); + if (!tagName) continue; + + // Set rating (philomena uses tags for this) + // These tags also have `data-tag-category="rating"` which we could use. + const safety = this.safetyMap.get(tagName); + if (safety != undefined) { + post.rating = safety; + // Don't add this as a tag. + continue; + } + + let category: TagCategory | undefined; + + // Categorize tags using data-tag-category + const dataTagCategory = el.getAttribute("data-tag-category"); + if (dataTagCategory) { + const mappedCategory = this.categoryMap.get(dataTagCategory); + if (mappedCategory) { + category = mappedCategory; + } + } + + // Artist tag categorize using prefixes + for (const prefix of this.artistPrefixes) { + if (tagName.includes(prefix + ":")) { + category = "artist"; + break; + } + } + + // Remove listed prefixes (removePrefixes) from tagName + for (const prefix of this.removePrefixes) { + if (tagName.includes(prefix + ":")) { + tagName = tagName.replace(prefix + ":", "").trim(); + + // Add suffix if configured + const suffix = this.removePrefixesSuffix.get(prefix); + if (suffix) { + tagName = tagName + suffix; + } + + break; + } + } + + // Push tag + if (tagName) { + const tag = new ScrapedTag(tagName, category); + post.tags.push(tag); + } + } + + // Set source + const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement); + post.sources = sourceEls.map((x) => x.href); + + result.tryAddPost(post); + + return result; + } +} diff --git a/src/engines/index.ts b/src/engines/index.ts index a46b9a6..1eab8a6 100644 --- a/src/engines/index.ts +++ b/src/engines/index.ts @@ -7,6 +7,7 @@ export { default as FurAffinity } from "./FurAffinity.js"; export { default as Gelbooru } from "./Gelbooru.js"; export { default as Inkbunny } from "./Inkbunny.js"; export { default as Moebooru } from "./Moebooru.js"; +export { default as Philomena } from "./Philomena.js"; export { default as Pixiv } from "./Pixiv.js"; export { default as Reddit } from "./Reddit.js"; export { default as Rule34us } from "./Rule34us.js";