diff --git a/README.md b/README.md index 8ef8009..6710342 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,66 @@ -# neo-scraper +# SzuruChrome -Scrape a booru-like post object from the DOM. +Chrome/Firefox extension which can import media from various sites into a [szurubooru](https://github.com/rr-/szurubooru) instance. ## Supported sites -| Engine | Sites | Features | Notes | -| -------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | -| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | -| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | -| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | -| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | -| inkbunny | inkbunny.net | content, rating, resolution, tags | | -| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | -| pixiv | pixiv.net, www.pixiv.net | content | | -| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | -| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | -| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | -| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | -| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | -| twitter | twitter.com, mobile.twitter.com | content | | -| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| fallback | | content | Tries to find the largest image or video on the current page. | +| Engine | Sites | Features | Notes | +| -------------- | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | +| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | +| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | +| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | +| inkbunny | inkbunny.net | content, rating, resolution, tags | | +| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | +| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, tags, source | | +| pixiv | pixiv.net, www.pixiv.net | content | | +| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | +| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | +| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | Hotlink protection currently prevents szurubooru from importing from this site. | +| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | +| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | +| twitter | twitter.com, mobile.twitter.com | content | | +| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| fallback | | content | Tries to find the largest image or video on the current page. | + +## Installation + +For development see [build/debug](#Builddebug). + +### Firefox + +Just download the .xpi from [release tab](https://github.com/neobooru/SzuruChrome/releases) and install it. + +### Chrome + +I can't create a valid .crx file, which means that you'll have to use [Chrome's "Load unpacked extension"](https://developer.chrome.com/docs/extensions/mv3/getstarted/#unpacked) feature. +You can download a .zip containing the needed files from the [release tab](https://github.com/neobooru/SzuruChrome/releases). Check [this stackoverflow answer](https://stackoverflow.com/a/38011386) if you want to get rid of the developer extension popup. + +## Build/debug + +```sh +# Install node modules +pnpm install + +# Live dev build (watch) +pnpm dev + +# Production build +pnpm build +``` + +After this you can load them in your browser. +Chrome: `chrome://extensions/` → "Load unpacked" → Select `./dist` folder +Firefox: `about:debugging` → "Load Temporary Add-on..." → Select any file in the `./dist` folder (it will load all files) + +See [the developer docs](./docs/DEVELOPER.md) for more information. + +## Planned features + +These are some things I might work on. Though honestly I don't expect this extension to change much anytime soon. + +- Finishing the initial version of this add-on (see visual design doc for all features) +- Right click any image on a page to import it +- Configurable list of tags to always ignore (e.g. tagme, highres) +- [low priority] Import artist commentary diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 16dd8ce..3f35a04 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -16,6 +16,7 @@ export default class NeoScraper { new engines.Gelbooru(), new engines.Inkbunny(), new engines.Moebooru(), + new engines.Philomena(), new engines.Pixiv(), new engines.Reddit(), new engines.Rule34us(), diff --git a/src/engines/Philomena.test.ts b/src/engines/Philomena.test.ts new file mode 100644 index 0000000..fbb16ba --- /dev/null +++ b/src/engines/Philomena.test.ts @@ -0,0 +1,13 @@ +import { scrapeUrl } from "../test/engine.js"; + +test("get derpibooru.org post 380315", async () => { + const pageUrl = "https://derpibooru.org/images/3153524"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("safe"); + expect(res.posts[0]).toHaveTag("yay"); + expect(res.posts[0]).toHaveTag("flutteryay"); + expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640"); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg"); +}); diff --git a/src/engines/Philomena.ts b/src/engines/Philomena.ts new file mode 100644 index 0000000..127eb33 --- /dev/null +++ b/src/engines/Philomena.ts @@ -0,0 +1,68 @@ +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; +import { TagCategory } from "../BooruTypes.js"; + +export default class Philomena extends ScrapeEngineBase { + name = "Philomena"; + features: ScrapeEngineFeature[] = ["content", "rating", "tags", "source"]; + notes = []; + supportedHosts = ["derpibooru.org", "trixiebooru.org", "ponybooru.org", "furbooru.org", "ponerpics.org", "manebooru.art", "twibooru.org"]; + + scrapeDocument(document: Document): ScrapeResult { + const result = new ScrapeResult(this.name); + const post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + const downloadEl = document.querySelector("div[id^='image_meta_'] > div:nth-child(4) > a:nth-child(4)") as HTMLAnchorElement; + post.contentUrl = downloadEl?.href; + + // Set content type + if (document.querySelector("video#image-display") != null) { + post.contentType = "video"; + } + + // Set default rating (Will be determined later) + post.rating = "unsafe"; + + // Set tags + const tagEls = Array.from(document.querySelectorAll("a.tag__name")).map((x) => x as HTMLLIElement); + // Workaround for forked builds of philomena + if (tagEls.length === 0) { + + } + + for (const el of tagEls) { + const tagName = (el.innerText); + let category: TagCategory | undefined; + + // Set rating (philomena uses tags for this) + switch (tagName) { + case "safe": + post.rating = "safe"; + break; + case "suggestive": + post.rating = "sketchy"; + break; + case "questionable": + post.rating = "sketchy"; + break; + case "explicit": + post.rating = "unsafe"; + break; + } + + if (tagName) { + const tag = new ScrapedTag(tagName, category); + post.tags.push(tag); + } + } + + // Set source + const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement); + post.sources = sourceEls.map((x) => x.href); + + result.tryAddPost(post); + + return result; + } +} diff --git a/src/engines/index.ts b/src/engines/index.ts index a46b9a6..1eab8a6 100644 --- a/src/engines/index.ts +++ b/src/engines/index.ts @@ -7,6 +7,7 @@ export { default as FurAffinity } from "./FurAffinity.js"; export { default as Gelbooru } from "./Gelbooru.js"; export { default as Inkbunny } from "./Inkbunny.js"; export { default as Moebooru } from "./Moebooru.js"; +export { default as Philomena } from "./Philomena.js"; export { default as Pixiv } from "./Pixiv.js"; export { default as Reddit } from "./Reddit.js"; export { default as Rule34us } from "./Rule34us.js";