Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge https://github.com/neobooru/neo-scraper

pull/19/head
Kaias Blackwood 2025-08-14 17:58:51 +00:00
parent e1ffab613b 51be6dfdcd
commit bf9f3b8e7a
10 changed files with 189 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
README.md
View File

@ -4,25 +4,28 @@ Scrape a booru-like post object from the DOM.
## Supported sites
| Engine | Sites | Features | Notes |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | |
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | |
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net, www.pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com | content | |
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |
| Engine | Sites | Features | Notes |
|---|---|---|---|---|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| bluesky | bsky.app | content | |
| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | |
| E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
| furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
| nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
| Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com, x.com | content | |
| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |
## Development
@ -34,5 +37,6 @@ pnpm install
pnpm build # or pnpm watch in another terminal
pnpm link --global
cd ../SzuruChrome
pnpm link --global neo-scraper
pnpm dev
```

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "neo-scraper",
"version": "0.10.0",
"version": "0.10.1",
"description": "Scrape a booru-like post object from the DOM.",
"type": "module",
"main": "dist/index.js",

4
src/ScrapeEngine.ts
View File

@ -32,7 +32,7 @@ export class ScrapedTag {
}
export class ScrapedNote {
constructor(public text: string, public polygon: number[][]) { }
constructor(public text: string, public polygon: number[][]) {}
}
export class ScrapeResult {
@ -95,7 +95,7 @@ export abstract class ScrapeEngineBase implements ScrapeEngine {
abstract supportedHosts: string[];
canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host) != -1;
return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
}
abstract scrapeDocument(document: Document): ScrapeResult;

2
src/engines/Danbooru.ts
View File

@ -6,7 +6,7 @@ export default class Danbooru extends ScrapeEngineBase {
name = "danbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = [];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

128
src/engines/FurAffinity.ts
View File

@ -1,64 +1,64 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { guessContentType, parseResolutionString } from "../Utility.js";
export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = [];
supportedHosts = ["www.furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// Could use the download button here, but as far as I can tell it doesn't matter.
const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement;
post.contentUrl = downloadEl?.src;
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set resolution
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
if (sizeDiv) {
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
post.resolution = resolution;
}
// Set rating
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
(x) => x as HTMLSpanElement
);
if (ratingEls.length > 0)
switch (ratingEls[0].innerText) {
case "General":
post.rating = "safe";
break;
case "Mature":
post.rating = "sketchy";
break;
case "Adult":
post.rating = "unsafe";
break;
}
// Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
.map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText;
if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist"));
}
result.tryAddPost(post);
return result;
}
}
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { guessContentType, parseResolutionString } from "../Utility.js";
export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = [];
supportedHosts = ["furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// Could use the download button here, but as far as I can tell it doesn't matter.
const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement;
post.contentUrl = downloadEl?.src;
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set resolution
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
if (sizeDiv) {
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
post.resolution = resolution;
}
// Set rating
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
(x) => x as HTMLSpanElement
);
if (ratingEls.length > 0)
switch (ratingEls[0].innerText) {
case "General":
post.rating = "safe";
break;
case "Mature":
post.rating = "sketchy";
break;
case "Adult":
post.rating = "unsafe";
break;
}
// Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
.map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist"));
}
result.tryAddPost(post);
return result;
}
}

168
src/engines/Gelbooru.test.ts
View File

@ -1,84 +1,84 @@
import { scrapeUrl } from "../test/engine.js";
test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
});
test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("maosen", "artist");
expect(res.posts[0]).toHaveTag("butterfly");
expect(res.posts[0]).toHaveResolution([720, 1103]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
test("get gelbooru.com post 6225495 with notes", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("teeth");
expect(res.posts[0]).toHaveTag("highres");
expect(res.posts[0]).toHaveTag("nintendo");
expect(res.posts[0]).toHaveTag("link");
expect(res.posts[0]).toHaveTag("ttanuu.");
expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
expect(res.posts[0]).toHaveNote("But no need to worry!");
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
expect(res.posts[0]).toHaveResolution([1600, 2200]);
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
});
test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});
test("get xbooru.com post 834083", async () => {
// Don't worry this isn't a cursed image.
const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
expect(res.posts[0]).toHaveTag("nurse");
expect(res.posts[0]).toHaveTag("standing");
expect(res.posts[0]).toHaveResolution([1024, 1878]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
});
import { scrapeUrl } from "../test/engine.js";
test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
});
test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("maosen", "artist");
expect(res.posts[0]).toHaveTag("butterfly");
expect(res.posts[0]).toHaveResolution([720, 1103]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
test("get gelbooru.com post 6225495 with notes", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("teeth");
expect(res.posts[0]).toHaveTag("highres");
expect(res.posts[0]).toHaveTag("nintendo");
expect(res.posts[0]).toHaveTag("link");
expect(res.posts[0]).toHaveTag("ttanuu.");
expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
expect(res.posts[0]).toHaveNote("But no need to worry!");
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
expect(res.posts[0]).toHaveResolution([1600, 2200]);
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
});
test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});
test("get xbooru.com post 834083", async () => {
// Don't worry this isn't a cursed image.
const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
expect(res.posts[0]).toHaveTag("nurse");
expect(res.posts[0]).toHaveTag("standing");
expect(res.posts[0]).toHaveResolution([1024, 1878]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
});

12
src/engines/Gelbooru.ts
View File

@ -13,7 +13,15 @@ export default class Gelbooru extends ScrapeEngineBase {
name = "gelbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = ["Supported features might vary between hosts."];
supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com"];
supportedHosts = [
"safebooru.org",
"gelbooru.com",
"rule34.xxx",
"tbib.org",
"xbooru.com",
"mspabooru.com",
"hypnohub.net",
];
private readonly classNameToCategoryMap: CategoryMap = {
"tag-type-copyright": "copyright",
@ -30,6 +38,8 @@ export default class Gelbooru extends ScrapeEngineBase {
case "rule34.xxx":
case "tbib.org":
case "xbooru.com":
case "mspabooru.com":
case "hypnohub.net":
version = Version.v020;
break;
case "gelbooru.com":

2
src/engines/Moebooru.ts
View File

@ -6,7 +6,7 @@ export default class Moebooru extends ScrapeEngineBase {
name = "moebooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = [];
supportedHosts = ["yande.re", "konachan.com"];
supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

2
src/engines/Pixiv.ts
View File

@ -25,7 +25,7 @@ export default class Pixiv extends ScrapeEngineBase {
name = "pixiv";
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = ["pixiv.net", "www.pixiv.net"];
supportedHosts = ["pixiv.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

2
src/engines/Zerochan.ts
View File

@ -6,7 +6,7 @@ export default class Zerochan extends ScrapeEngineBase {
name = "zerochan";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."];
supportedHosts = ["www.zerochan.net"];
supportedHosts = ["zerochan.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);