From 8b89e7b527b49d57db19ffcde89392f584209039 Mon Sep 17 00:00:00 2001 From: GlobalGit3 <137680273+GlobalGit3@users.noreply.github.com> Date: Sat, 1 Jul 2023 08:17:53 -0500 Subject: [PATCH 1/9] Added Philomena * Philomena support --- README.md | 82 ++++++++++++++++++++++++++--------- src/NeoScraper.ts | 1 + src/engines/Philomena.test.ts | 13 ++++++ src/engines/Philomena.ts | 68 +++++++++++++++++++++++++++++ src/engines/index.ts | 1 + 5 files changed, 145 insertions(+), 20 deletions(-) create mode 100644 src/engines/Philomena.test.ts create mode 100644 src/engines/Philomena.ts diff --git a/README.md b/README.md index 8ef8009..6710342 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,66 @@ -# neo-scraper +# SzuruChrome -Scrape a booru-like post object from the DOM. +Chrome/Firefox extension which can import media from various sites into a [szurubooru](https://github.com/rr-/szurubooru) instance. ## Supported sites -| Engine | Sites | Features | Notes | -| -------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | -| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | -| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | -| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | -| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | -| inkbunny | inkbunny.net | content, rating, resolution, tags | | -| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | -| pixiv | pixiv.net, www.pixiv.net | content | | -| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | -| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | -| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | -| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | -| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | -| twitter | twitter.com, mobile.twitter.com | content | | -| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| fallback | | content | Tries to find the largest image or video on the current page. | +| Engine | Sites | Features | Notes | +| -------------- | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | +| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | +| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | +| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | +| inkbunny | inkbunny.net | content, rating, resolution, tags | | +| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | +| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, tags, source | | +| pixiv | pixiv.net, www.pixiv.net | content | | +| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | +| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | +| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | Hotlink protection currently prevents szurubooru from importing from this site. | +| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | +| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | +| twitter | twitter.com, mobile.twitter.com | content | | +| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| fallback | | content | Tries to find the largest image or video on the current page. | + +## Installation + +For development see [build/debug](#Builddebug). + +### Firefox + +Just download the .xpi from [release tab](https://github.com/neobooru/SzuruChrome/releases) and install it. + +### Chrome + +I can't create a valid .crx file, which means that you'll have to use [Chrome's "Load unpacked extension"](https://developer.chrome.com/docs/extensions/mv3/getstarted/#unpacked) feature. +You can download a .zip containing the needed files from the [release tab](https://github.com/neobooru/SzuruChrome/releases). Check [this stackoverflow answer](https://stackoverflow.com/a/38011386) if you want to get rid of the developer extension popup. + +## Build/debug + +```sh +# Install node modules +pnpm install + +# Live dev build (watch) +pnpm dev + +# Production build +pnpm build +``` + +After this you can load them in your browser. +Chrome: `chrome://extensions/` → "Load unpacked" → Select `./dist` folder +Firefox: `about:debugging` → "Load Temporary Add-on..." → Select any file in the `./dist` folder (it will load all files) + +See [the developer docs](./docs/DEVELOPER.md) for more information. + +## Planned features + +These are some things I might work on. Though honestly I don't expect this extension to change much anytime soon. + +- Finishing the initial version of this add-on (see visual design doc for all features) +- Right click any image on a page to import it +- Configurable list of tags to always ignore (e.g. tagme, highres) +- [low priority] Import artist commentary diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 16dd8ce..3f35a04 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -16,6 +16,7 @@ export default class NeoScraper { new engines.Gelbooru(), new engines.Inkbunny(), new engines.Moebooru(), + new engines.Philomena(), new engines.Pixiv(), new engines.Reddit(), new engines.Rule34us(), diff --git a/src/engines/Philomena.test.ts b/src/engines/Philomena.test.ts new file mode 100644 index 0000000..fbb16ba --- /dev/null +++ b/src/engines/Philomena.test.ts @@ -0,0 +1,13 @@ +import { scrapeUrl } from "../test/engine.js"; + +test("get derpibooru.org post 380315", async () => { + const pageUrl = "https://derpibooru.org/images/3153524"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("safe"); + expect(res.posts[0]).toHaveTag("yay"); + expect(res.posts[0]).toHaveTag("flutteryay"); + expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640"); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg"); +}); diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts new file mode 100644 index 0000000..127eb33 --- /dev/null +++ b/src/engines/Philomena.ts @@ -0,0 +1,68 @@ +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; +import { TagCategory } from "../BooruTypes.js"; + +export default class Philomena extends ScrapeEngineBase { + name = "Philomena"; + features: ScrapeEngineFeature[] = ["content", "rating", "tags", "source"]; + notes = []; + supportedHosts = ["derpibooru.org", "trixiebooru.org", "ponybooru.org", "furbooru.org", "ponerpics.org", "manebooru.art", "twibooru.org"]; + + scrapeDocument(document: Document): ScrapeResult { + const result = new ScrapeResult(this.name); + const post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + const downloadEl = document.querySelector("div[id^='image_meta_'] > div:nth-child(4) > a:nth-child(4)") as HTMLAnchorElement; + post.contentUrl = downloadEl?.href; + + // Set content type + if (document.querySelector("video#image-display") != null) { + post.contentType = "video"; + } + + // Set default rating (Will be determined later) + post.rating = "unsafe"; + + // Set tags + const tagEls = Array.from(document.querySelectorAll("a.tag__name")).map((x) => x as HTMLLIElement); + // Workaround for forked builds of philomena + if (tagEls.length === 0) { + + } + + for (const el of tagEls) { + const tagName = (el.innerText); + let category: TagCategory | undefined; + + // Set rating (philomena uses tags for this) + switch (tagName) { + case "safe": + post.rating = "safe"; + break; + case "suggestive": + post.rating = "sketchy"; + break; + case "questionable": + post.rating = "sketchy"; + break; + case "explicit": + post.rating = "unsafe"; + break; + } + + if (tagName) { + const tag = new ScrapedTag(tagName, category); + post.tags.push(tag); + } + } + + // Set source + const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement); + post.sources = sourceEls.map((x) => x.href); + + result.tryAddPost(post); + + return result; + } +} diff --git a/src/engines/index.ts b/src/engines/index.ts index a46b9a6..1eab8a6 100644 --- a/src/engines/index.ts +++ b/src/engines/index.ts @@ -7,6 +7,7 @@ export { default as FurAffinity } from "./FurAffinity.js"; export { default as Gelbooru } from "./Gelbooru.js"; export { default as Inkbunny } from "./Inkbunny.js"; export { default as Moebooru } from "./Moebooru.js"; +export { default as Philomena } from "./Philomena.js"; export { default as Pixiv } from "./Pixiv.js"; export { default as Reddit } from "./Reddit.js"; export { default as Rule34us } from "./Rule34us.js"; From e10baedddc1aedee8c2b3ccc45619d714b03c346 Mon Sep 17 00:00:00 2001 From: GlobalGit3 <137680273+GlobalGit3@users.noreply.github.com> Date: Sat, 1 Jul 2023 08:30:07 -0500 Subject: [PATCH 2/9] Update Philomena.test.ts Fixed a small oversight. No worries with the old id. --- src/engines/Philomena.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/Philomena.test.ts b/src/engines/Philomena.test.ts index fbb16ba..8af0b2a 100644 --- a/src/engines/Philomena.test.ts +++ b/src/engines/Philomena.test.ts @@ -1,6 +1,6 @@ import { scrapeUrl } from "../test/engine.js"; -test("get derpibooru.org post 380315", async () => { +test("get derpibooru.org post 3153524", async () => { const pageUrl = "https://derpibooru.org/images/3153524"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); From b9ae6a1e8acb2f9e15631a5d0bcb9832ccb51771 Mon Sep 17 00:00:00 2001 From: GlobalGit3 <137680273+GlobalGit3@users.noreply.github.com> Date: Sat, 1 Jul 2023 08:32:35 -0500 Subject: [PATCH 3/9] Update README.md Used the wrong readme, oopsies. --- README.md | 45 ++------------------------------------------- 1 file changed, 2 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 6710342..24ebb67 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# SzuruChrome +# neo-scraper -Chrome/Firefox extension which can import media from various sites into a [szurubooru](https://github.com/rr-/szurubooru) instance. +Scrape a booru-like post object from the DOM. ## Supported sites @@ -23,44 +23,3 @@ Chrome/Firefox extension which can import media from various sites into a [szuru | twitter | twitter.com, mobile.twitter.com | content | | | zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | | fallback | | content | Tries to find the largest image or video on the current page. | - -## Installation - -For development see [build/debug](#Builddebug). - -### Firefox - -Just download the .xpi from [release tab](https://github.com/neobooru/SzuruChrome/releases) and install it. - -### Chrome - -I can't create a valid .crx file, which means that you'll have to use [Chrome's "Load unpacked extension"](https://developer.chrome.com/docs/extensions/mv3/getstarted/#unpacked) feature. -You can download a .zip containing the needed files from the [release tab](https://github.com/neobooru/SzuruChrome/releases). Check [this stackoverflow answer](https://stackoverflow.com/a/38011386) if you want to get rid of the developer extension popup. - -## Build/debug - -```sh -# Install node modules -pnpm install - -# Live dev build (watch) -pnpm dev - -# Production build -pnpm build -``` - -After this you can load them in your browser. -Chrome: `chrome://extensions/` → "Load unpacked" → Select `./dist` folder -Firefox: `about:debugging` → "Load Temporary Add-on..." → Select any file in the `./dist` folder (it will load all files) - -See [the developer docs](./docs/DEVELOPER.md) for more information. - -## Planned features - -These are some things I might work on. Though honestly I don't expect this extension to change much anytime soon. - -- Finishing the initial version of this add-on (see visual design doc for all features) -- Right click any image on a page to import it -- Configurable list of tags to always ignore (e.g. tagme, highres) -- [low priority] Import artist commentary From 3407ae2a8ccffc1ff702f92f0e379fde67d83ae9 Mon Sep 17 00:00:00 2001 From: GlobalGit3 <137680273+GlobalGit3@users.noreply.github.com> Date: Sat, 1 Jul 2023 14:14:44 -0500 Subject: [PATCH 4/9] Update Philomena.ts * Removed useless check --- src/engines/Philomena.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts index 127eb33..16a52f7 100644 --- a/src/engines/Philomena.ts +++ b/src/engines/Philomena.ts @@ -26,10 +26,6 @@ export default class Philomena extends ScrapeEngineBase { // Set tags const tagEls = Array.from(document.querySelectorAll("a.tag__name")).map((x) => x as HTMLLIElement); - // Workaround for forked builds of philomena - if (tagEls.length === 0) { - - } for (const el of tagEls) { const tagName = (el.innerText); From 4c0c790c0ce0fc8eb9e375518b63e71749496932 Mon Sep 17 00:00:00 2001 From: GlobalGit3 <137680273+GlobalGit3@users.noreply.github.com> Date: Sat, 1 Jul 2023 14:24:08 -0500 Subject: [PATCH 5/9] Attempt to 'prettier' code * Hopefully this is 'prettier' --- src/NeoScraper.ts | 2 +- src/engines/Philomena.ts | 58 +++++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 3f35a04..b07bfb1 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -16,7 +16,7 @@ export default class NeoScraper { new engines.Gelbooru(), new engines.Inkbunny(), new engines.Moebooru(), - new engines.Philomena(), + new engines.Philomena(), new engines.Pixiv(), new engines.Reddit(), new engines.Rule34us(), diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts index 16a52f7..430a5c8 100644 --- a/src/engines/Philomena.ts +++ b/src/engines/Philomena.ts @@ -5,7 +5,15 @@ export default class Philomena extends ScrapeEngineBase { name = "Philomena"; features: ScrapeEngineFeature[] = ["content", "rating", "tags", "source"]; notes = []; - supportedHosts = ["derpibooru.org", "trixiebooru.org", "ponybooru.org", "furbooru.org", "ponerpics.org", "manebooru.art", "twibooru.org"]; + supportedHosts = [ + "derpibooru.org", + "trixiebooru.org", + "ponybooru.org", + "furbooru.org", + "ponerpics.org", + "manebooru.art", + "twibooru.org", + ]; scrapeDocument(document: Document): ScrapeResult { const result = new ScrapeResult(this.name); @@ -13,7 +21,9 @@ export default class Philomena extends ScrapeEngineBase { post.pageUrl = document.location.href; // Set image url - const downloadEl = document.querySelector("div[id^='image_meta_'] > div:nth-child(4) > a:nth-child(4)") as HTMLAnchorElement; + const downloadEl = document.querySelector( + "div[id^='image_meta_'] > div:nth-child(4) > a:nth-child(4)" + ) as HTMLAnchorElement; post.contentUrl = downloadEl?.href; // Set content type @@ -22,37 +32,37 @@ export default class Philomena extends ScrapeEngineBase { } // Set default rating (Will be determined later) - post.rating = "unsafe"; + post.rating = "unsafe"; // Set tags const tagEls = Array.from(document.querySelectorAll("a.tag__name")).map((x) => x as HTMLLIElement); - - for (const el of tagEls) { - const tagName = (el.innerText); + + for (const el of tagEls) { + const tagName = el.innerText; let category: TagCategory | undefined; - - // Set rating (philomena uses tags for this) - switch (tagName) { - case "safe": - post.rating = "safe"; - break; - case "suggestive": - post.rating = "sketchy"; - break; - case "questionable": - post.rating = "sketchy"; - break; - case "explicit": - post.rating = "unsafe"; - break; - } - + + // Set rating (philomena uses tags for this) + switch (tagName) { + case "safe": + post.rating = "safe"; + break; + case "suggestive": + post.rating = "sketchy"; + break; + case "questionable": + post.rating = "sketchy"; + break; + case "explicit": + post.rating = "unsafe"; + break; + } + if (tagName) { const tag = new ScrapedTag(tagName, category); post.tags.push(tag); } } - + // Set source const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement); post.sources = sourceEls.map((x) => x.href); From 580e308f8f13f4c1bff8a7a87af1f6244af0627b Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Sun, 2 Jul 2023 00:20:12 +0200 Subject: [PATCH 6/9] Remove safety tags from list and add categories --- src/engines/Philomena.test.ts | 3 +- src/engines/Philomena.ts | 57 +++++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/engines/Philomena.test.ts b/src/engines/Philomena.test.ts index 8af0b2a..9586ade 100644 --- a/src/engines/Philomena.test.ts +++ b/src/engines/Philomena.test.ts @@ -4,7 +4,8 @@ test("get derpibooru.org post 3153524", async () => { const pageUrl = "https://derpibooru.org/images/3153524"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); - expect(res.posts[0]).toHaveTag("safe"); + expect(res.posts[0].rating).toBe("safe"); + expect(res.posts[0]).not.toHaveTag("safe"); expect(res.posts[0]).toHaveTag("yay"); expect(res.posts[0]).toHaveTag("flutteryay"); expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640"); diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts index 430a5c8..c79f684 100644 --- a/src/engines/Philomena.ts +++ b/src/engines/Philomena.ts @@ -1,5 +1,5 @@ import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; -import { TagCategory } from "../BooruTypes.js"; +import { SafetyRating, TagCategory } from "../BooruTypes.js"; export default class Philomena extends ScrapeEngineBase { name = "Philomena"; @@ -14,6 +14,17 @@ export default class Philomena extends ScrapeEngineBase { "manebooru.art", "twibooru.org", ]; + safetyMap = new Map([ + ["safe", "safe"], + ["suggestive", "sketchy"], + ["questionable", "sketchy"], + ["explicit", "unsafe"], + ]); + categoryMap = new Map([ + ["character", "character"], + ["species", "species"], + ["oc", "character"], + ]); scrapeDocument(document: Document): ScrapeResult { const result = new ScrapeResult(this.name); @@ -21,9 +32,7 @@ export default class Philomena extends ScrapeEngineBase { post.pageUrl = document.location.href; // Set image url - const downloadEl = document.querySelector( - "div[id^='image_meta_'] > div:nth-child(4) > a:nth-child(4)" - ) as HTMLAnchorElement; + const downloadEl = document.querySelector("a[title='Download (no tags in filename)']") as HTMLAnchorElement; post.contentUrl = downloadEl?.href; // Set content type @@ -35,26 +44,34 @@ export default class Philomena extends ScrapeEngineBase { post.rating = "unsafe"; // Set tags - const tagEls = Array.from(document.querySelectorAll("a.tag__name")).map((x) => x as HTMLLIElement); + const tagEls = Array.from(document.querySelectorAll(".tag.dropdown")).map((x) => x as HTMLSpanElement); for (const el of tagEls) { - const tagName = el.innerText; - let category: TagCategory | undefined; + let tagName = el.getAttribute("data-tag-name"); + if (!tagName) continue; // Set rating (philomena uses tags for this) - switch (tagName) { - case "safe": - post.rating = "safe"; - break; - case "suggestive": - post.rating = "sketchy"; - break; - case "questionable": - post.rating = "sketchy"; - break; - case "explicit": - post.rating = "unsafe"; - break; + // These tags also have `data-tag-category="rating"` which we could use. + const safety = this.safetyMap.get(tagName); + if (safety != undefined) { + post.rating = safety; + // Don't add this as a tag. + continue; + } + + let category: TagCategory | undefined; + const artistMatches = tagName.match("artist:(.*)"); + if (artistMatches && artistMatches.length == 2) { + tagName = artistMatches[1]; + category = "artist"; + } + + const dataTagCategory = el.getAttribute("data-tag-category"); + if (dataTagCategory) { + const mappedCategory = this.categoryMap.get(dataTagCategory); + if (mappedCategory) { + category = mappedCategory; + } } if (tagName) { From 8d215729f318dfff49315a496ba64d14c4d79cae Mon Sep 17 00:00:00 2001 From: GlobalGit3 <137680273+GlobalGit3@users.noreply.github.com> Date: Sun, 2 Jul 2023 01:49:40 -0500 Subject: [PATCH 7/9] Map and Arrays for prefixes Categorize tags and remove prefixes using Maps and Arrays. A quick write-up, but works for the time being to get an idea for what is wanted. Due to the data-tag-category's 'origin' being used for more than artist's names, prefixes are used to categorize the artist tag. There are also other prefixes such as 'editor' that are the same as 'artist'. This should be no problem as all artists' tags start with the 'artist' or 'editor' prefix. Suffixes only applies when removing a prefix, this will probably be changed later to apply without removing the prefix. It was coded to give an idea of wanting to change tagNames from "oc:originalcharacter" to "originalcharacter_(oc)". --- src/engines/Philomena.ts | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts index c79f684..53e8c84 100644 --- a/src/engines/Philomena.ts +++ b/src/engines/Philomena.ts @@ -14,6 +14,9 @@ export default class Philomena extends ScrapeEngineBase { "manebooru.art", "twibooru.org", ]; + removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"]; + removeprefixesSuffix = new Map([["oc", "_(oc)"]]); + artistPrefixes = ["artist", "editor"]; safetyMap = new Map([ ["safe", "safe"], ["suggestive", "sketchy"], @@ -24,6 +27,7 @@ export default class Philomena extends ScrapeEngineBase { ["character", "character"], ["species", "species"], ["oc", "character"], + ["content-official", "copyright"], ]); scrapeDocument(document: Document): ScrapeResult { @@ -60,12 +64,8 @@ export default class Philomena extends ScrapeEngineBase { } let category: TagCategory | undefined; - const artistMatches = tagName.match("artist:(.*)"); - if (artistMatches && artistMatches.length == 2) { - tagName = artistMatches[1]; - category = "artist"; - } + // Categorize tags using data-tag-category const dataTagCategory = el.getAttribute("data-tag-category"); if (dataTagCategory) { const mappedCategory = this.categoryMap.get(dataTagCategory); @@ -74,6 +74,30 @@ export default class Philomena extends ScrapeEngineBase { } } + // Artist tag categorize using prefixes + for (const prefix of this.artistPrefixes) { + if (tagName.includes(prefix + ":")) { + category = "artist"; + break; + } + } + + // Remove listed prefixes (removePrefixes) from tagName + for (const prefix of this.removePrefixes) { + if (tagName.includes(prefix + ":")) { + tagName = tagName.replace(prefix + ":", "").trim(); + + // Add suffix if configured + const suffix = this.removeprefixesSuffix.get(prefix); + if (suffix) { + tagName = tagName + suffix; + } + + break; + } + } + + // Push tag if (tagName) { const tag = new ScrapedTag(tagName, category); post.tags.push(tag); From e208d6969588d1f2deaf358dd809f8c52465b462 Mon Sep 17 00:00:00 2001 From: GlobalGit3 <137680273+GlobalGit3@users.noreply.github.com> Date: Mon, 3 Jul 2023 16:43:39 -0500 Subject: [PATCH 8/9] Updated Features & Added resolution. Resolution now supported. Feature list was updated to reflect both resolution and tag category being used now. --- README.md | 38 +++++++++++++++++++------------------- src/engines/Philomena.ts | 9 ++++++++- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 24ebb67..72f25a8 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,22 @@ Scrape a booru-like post object from the DOM. ## Supported sites -| Engine | Sites | Features | Notes | -| -------------- | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | -| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | -| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | -| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | -| inkbunny | inkbunny.net | content, rating, resolution, tags | | -| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | -| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, tags, source | | -| pixiv | pixiv.net, www.pixiv.net | content | | -| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | -| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | -| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | Hotlink protection currently prevents szurubooru from importing from this site. | -| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | -| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | -| twitter | twitter.com, mobile.twitter.com | content | | -| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| fallback | | content | Tries to find the largest image or video on the current page. | +| Engine | Sites | Features | Notes | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | +| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | +| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | +| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | +| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | +| inkbunny | inkbunny.net | content, rating, resolution, tags | | +| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | +| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | | +| pixiv | pixiv.net, www.pixiv.net | content | | +| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | +| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | +| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | +| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | +| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | +| twitter | twitter.com, mobile.twitter.com | content | | +| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| fallback | | content | Tries to find the largest image or video on the current page. | \ No newline at end of file diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts index 53e8c84..5d21949 100644 --- a/src/engines/Philomena.ts +++ b/src/engines/Philomena.ts @@ -3,7 +3,7 @@ import { SafetyRating, TagCategory } from "../BooruTypes.js"; export default class Philomena extends ScrapeEngineBase { name = "Philomena"; - features: ScrapeEngineFeature[] = ["content", "rating", "tags", "source"]; + features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"]; notes = []; supportedHosts = [ "derpibooru.org", @@ -47,6 +47,13 @@ export default class Philomena extends ScrapeEngineBase { // Set default rating (Will be determined later) post.rating = "unsafe"; + // Set resolution + const resolutionEl = ((document.querySelector("span[class='image-size']"))?.innerText).split(" ")[0]; + const [width, height] = (resolutionEl?.replace(" ", "").split("x") || []).map(Number); + if (!isNaN(width) && !isNaN(height)) { + post.resolution = [width, height]; + } + // Set tags const tagEls = Array.from(document.querySelectorAll(".tag.dropdown")).map((x) => x as HTMLSpanElement); From 5beea8cc696e8031682745e4f6033540af13eb42 Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Wed, 5 Jul 2023 13:04:41 +0200 Subject: [PATCH 9/9] Small fixes and add extra test --- src/engines/Philomena.test.ts | 16 ++++++++++++++++ src/engines/Philomena.ts | 15 +++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/engines/Philomena.test.ts b/src/engines/Philomena.test.ts index 9586ade..155a6e9 100644 --- a/src/engines/Philomena.test.ts +++ b/src/engines/Philomena.test.ts @@ -9,6 +9,22 @@ test("get derpibooru.org post 3153524", async () => { expect(res.posts[0]).toHaveTag("yay"); expect(res.posts[0]).toHaveTag("flutteryay"); expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640"); + expect(res.posts[0]).toHaveResolution([4000, 4000]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg"); }); + +test("get derpibooru.org post 3157356", async () => { + const pageUrl = "https://derpibooru.org/images/3157356?q=oc"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].rating).toBe("safe"); + expect(res.posts[0]).not.toHaveTag("safe"); + expect(res.posts[0]).toHaveTag("oc"); + expect(res.posts[0]).toHaveTag("serra20", "artist"); + expect(res.posts[0]).toHaveTag("pony", "species"); + expect(res.posts[0]).toHaveTag("eris_(oc)", "character"); + expect(res.posts[0]).toHaveResolution([960, 960]); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/5/3157356.png"); +}); diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts index 5d21949..a8d5c87 100644 --- a/src/engines/Philomena.ts +++ b/src/engines/Philomena.ts @@ -1,5 +1,6 @@ import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { SafetyRating, TagCategory } from "../BooruTypes.js"; +import { parseResolutionString } from "../Utility.js"; export default class Philomena extends ScrapeEngineBase { name = "Philomena"; @@ -15,7 +16,7 @@ export default class Philomena extends ScrapeEngineBase { "twibooru.org", ]; removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"]; - removeprefixesSuffix = new Map([["oc", "_(oc)"]]); + removePrefixesSuffix = new Map([["oc", "_(oc)"]]); artistPrefixes = ["artist", "editor"]; safetyMap = new Map([ ["safe", "safe"], @@ -48,10 +49,12 @@ export default class Philomena extends ScrapeEngineBase { post.rating = "unsafe"; // Set resolution - const resolutionEl = ((document.querySelector("span[class='image-size']"))?.innerText).split(" ")[0]; - const [width, height] = (resolutionEl?.replace(" ", "").split("x") || []).map(Number); - if (!isNaN(width) && !isNaN(height)) { - post.resolution = [width, height]; + const resStr = (document.querySelector("span[class='image-size']"))?.innerText?.split(" ")[0]; + if (resStr) { + const res = parseResolutionString(resStr); + if (res) { + post.resolution = res; + } } // Set tags @@ -95,7 +98,7 @@ export default class Philomena extends ScrapeEngineBase { tagName = tagName.replace(prefix + ":", "").trim(); // Add suffix if configured - const suffix = this.removeprefixesSuffix.get(prefix); + const suffix = this.removePrefixesSuffix.get(prefix); if (suffix) { tagName = tagName + suffix; }