Kaias Blackwood 2025-08-14 17:58:51 +00:00
commit bf9f3b8e7a
10 changed files with 189 additions and 175 deletions

View File

@ -4,25 +4,28 @@ Scrape a booru-like post object from the DOM.
## Supported sites ## Supported sites
| Engine | Sites | Features | Notes | | Engine | Sites | Features | Notes |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | |---|---|---|---|---|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | | animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | | bluesky | bsky.app | content | |
| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | | | danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | | e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | | E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | | | furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | |
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | | gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | | | inkbunny | inkbunny.net | content, rating, resolution, tags | |
| pixiv | pixiv.net, www.pixiv.net | content | | | moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | | nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | | Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | | pixiv | pixiv.net | content | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | | reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | | rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| twitter | twitter.com, mobile.twitter.com | content | | | sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | | shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| fallback | | content | Tries to find the largest image or video on the current page. | | shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com, x.com | content | |
| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |
## Development ## Development
@ -34,5 +37,6 @@ pnpm install
pnpm build # or pnpm watch in another terminal pnpm build # or pnpm watch in another terminal
pnpm link --global pnpm link --global
cd ../SzuruChrome cd ../SzuruChrome
pnpm link --global neo-scraper
pnpm dev pnpm dev
``` ```

View File

@ -1,6 +1,6 @@
{ {
"name": "neo-scraper", "name": "neo-scraper",
"version": "0.10.0", "version": "0.10.1",
"description": "Scrape a booru-like post object from the DOM.", "description": "Scrape a booru-like post object from the DOM.",
"type": "module", "type": "module",
"main": "dist/index.js", "main": "dist/index.js",

View File

@ -32,7 +32,7 @@ export class ScrapedTag {
} }
export class ScrapedNote { export class ScrapedNote {
constructor(public text: string, public polygon: number[][]) { } constructor(public text: string, public polygon: number[][]) {}
} }
export class ScrapeResult { export class ScrapeResult {
@ -95,7 +95,7 @@ export abstract class ScrapeEngineBase implements ScrapeEngine {
abstract supportedHosts: string[]; abstract supportedHosts: string[];
canImport(url: Location): boolean { canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host) != -1; return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
} }
abstract scrapeDocument(document: Document): ScrapeResult; abstract scrapeDocument(document: Document): ScrapeResult;

View File

@ -6,7 +6,7 @@ export default class Danbooru extends ScrapeEngineBase {
name = "danbooru"; name = "danbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = []; notes = [];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"]; supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

View File

@ -1,64 +1,64 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { guessContentType, parseResolutionString } from "../Utility.js"; import { guessContentType, parseResolutionString } from "../Utility.js";
export default class FurAffinity extends ScrapeEngineBase { export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity"; name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = []; notes = [];
supportedHosts = ["www.furaffinity.net"]; supportedHosts = ["furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);
const post = new ScrapedPost(); const post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
// Could use the download button here, but as far as I can tell it doesn't matter. // Could use the download button here, but as far as I can tell it doesn't matter.
const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement; const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement;
post.contentUrl = downloadEl?.src; post.contentUrl = downloadEl?.src;
// Set content type // Set content type
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);
// Set resolution // Set resolution
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []); const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size"); const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
if (sizeDiv) { if (sizeDiv) {
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText); const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
post.resolution = resolution; post.resolution = resolution;
} }
// Set rating // Set rating
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map( const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
(x) => x as HTMLSpanElement (x) => x as HTMLSpanElement
); );
if (ratingEls.length > 0) if (ratingEls.length > 0)
switch (ratingEls[0].innerText) { switch (ratingEls[0].innerText) {
case "General": case "General":
post.rating = "safe"; post.rating = "safe";
break; break;
case "Mature": case "Mature":
post.rating = "sketchy"; post.rating = "sketchy";
break; break;
case "Adult": case "Adult":
post.rating = "unsafe"; post.rating = "unsafe";
break; break;
} }
// Set tags // Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags")) post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
.map((x) => (x as HTMLSpanElement).innerText) .map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x)); .map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist // Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText; const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
if (submissionUser) { if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist")); post.tags.push(new ScrapedTag(submissionUser, "artist"));
} }
result.tryAddPost(post); result.tryAddPost(post);
return result; return result;
} }
} }

View File

@ -1,84 +1,84 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine.js";
test("get safebooru.org post 2871532", async () => { test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532"; const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist"); expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character"); expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt"); expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt"); expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt"); expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair"); expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]); expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe"); expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain( expect(res.posts[0].contentUrl).toContain(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg" "https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
); );
}); });
test("get gelbooru.com post 4925055", async () => { test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba"; const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("maosen", "artist"); expect(res.posts[0]).toHaveTag("maosen", "artist");
expect(res.posts[0]).toHaveTag("butterfly"); expect(res.posts[0]).toHaveTag("butterfly");
expect(res.posts[0]).toHaveResolution([720, 1103]); expect(res.posts[0]).toHaveResolution([720, 1103]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958"); expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy"); expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
}); });
test("get gelbooru.com post 6225495 with notes", async () => { test("get gelbooru.com post 6225495 with notes", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495"; const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("hair_up"); expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("hair_up"); expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("teeth"); expect(res.posts[0]).toHaveTag("teeth");
expect(res.posts[0]).toHaveTag("highres"); expect(res.posts[0]).toHaveTag("highres");
expect(res.posts[0]).toHaveTag("nintendo"); expect(res.posts[0]).toHaveTag("nintendo");
expect(res.posts[0]).toHaveTag("link"); expect(res.posts[0]).toHaveTag("link");
expect(res.posts[0]).toHaveTag("ttanuu."); expect(res.posts[0]).toHaveTag("ttanuu.");
expect(res.posts[0]).toHaveNote("I'm fine, more or less."); expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
expect(res.posts[0]).toHaveNote("But no need to worry!"); expect(res.posts[0]).toHaveNote("But no need to worry!");
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!"); expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
expect(res.posts[0]).toHaveResolution([1600, 2200]); expect(res.posts[0]).toHaveResolution([1600, 2200]);
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816"); expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy"); expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png"); expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
}); });
test("get rule34.xxx post 2740994", async () => { test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994"; const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright"); expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright"); expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character"); expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
expect(res.posts[0]).toHaveTag("blue_hair"); expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo"); expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]); expect(res.posts[0]).toHaveResolution([600, 800]);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe"); expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg"); expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
}); });
test("get xbooru.com post 834083", async () => { test("get xbooru.com post 834083", async () => {
// Don't worry this isn't a cursed image. // Don't worry this isn't a cursed image.
const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083"; const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright"); expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
expect(res.posts[0]).toHaveTag("nurse_redheart", "character"); expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
expect(res.posts[0]).toHaveTag("nurse"); expect(res.posts[0]).toHaveTag("nurse");
expect(res.posts[0]).toHaveTag("standing"); expect(res.posts[0]).toHaveTag("standing");
expect(res.posts[0]).toHaveResolution([1024, 1878]); expect(res.posts[0]).toHaveResolution([1024, 1878]);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe"); expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png"); expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
}); });

View File

@ -13,7 +13,15 @@ export default class Gelbooru extends ScrapeEngineBase {
name = "gelbooru"; name = "gelbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = ["Supported features might vary between hosts."]; notes = ["Supported features might vary between hosts."];
supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com"]; supportedHosts = [
"safebooru.org",
"gelbooru.com",
"rule34.xxx",
"tbib.org",
"xbooru.com",
"mspabooru.com",
"hypnohub.net",
];
private readonly classNameToCategoryMap: CategoryMap = { private readonly classNameToCategoryMap: CategoryMap = {
"tag-type-copyright": "copyright", "tag-type-copyright": "copyright",
@ -30,6 +38,8 @@ export default class Gelbooru extends ScrapeEngineBase {
case "rule34.xxx": case "rule34.xxx":
case "tbib.org": case "tbib.org":
case "xbooru.com": case "xbooru.com":
case "mspabooru.com":
case "hypnohub.net":
version = Version.v020; version = Version.v020;
break; break;
case "gelbooru.com": case "gelbooru.com":

View File

@ -6,7 +6,7 @@ export default class Moebooru extends ScrapeEngineBase {
name = "moebooru"; name = "moebooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = []; notes = [];
supportedHosts = ["yande.re", "konachan.com"]; supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

View File

@ -25,7 +25,7 @@ export default class Pixiv extends ScrapeEngineBase {
name = "pixiv"; name = "pixiv";
features: ScrapeEngineFeature[] = ["content"]; features: ScrapeEngineFeature[] = ["content"];
notes = []; notes = [];
supportedHosts = ["pixiv.net", "www.pixiv.net"]; supportedHosts = ["pixiv.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

View File

@ -6,7 +6,7 @@ export default class Zerochan extends ScrapeEngineBase {
name = "zerochan"; name = "zerochan";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"]; features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."]; notes = ["Rating is assumed to be safe."];
supportedHosts = ["www.zerochan.net"]; supportedHosts = ["zerochan.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);