Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge https://github.com/neobooru/neo-scraper

pull/19/head
Kaias Blackwood 2025-08-14 17:58:51 +00:00
parent e1ffab613b 51be6dfdcd
commit bf9f3b8e7a
10 changed files with 189 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
README.md
View File

@ -5,23 +5,26 @@ Scrape a booru-like post object from the DOM.
## Supported sites ## Supported sites
| Engine | Sites | Features | Notes | | Engine | Sites | Features | Notes |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | |---|---|---|---|---|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | | animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | | bluesky | bsky.app | content | |
| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | | | e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | |
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | | E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | | furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | | | inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | | moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | | | nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
| pixiv | pixiv.net, www.pixiv.net | content | | | Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | | reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | | rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | | sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | | shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | | shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com | content | | | twitter | twitter.com, mobile.twitter.com, x.com | content | |
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | | zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. | | fallback | | content | Tries to find the largest image or video on the current page. |
## Development ## Development
@ -34,5 +37,6 @@ pnpm install
pnpm build # or pnpm watch in another terminal pnpm build # or pnpm watch in another terminal
pnpm link --global pnpm link --global
cd ../SzuruChrome cd ../SzuruChrome
pnpm link --global neo-scraper
pnpm dev pnpm dev
``` ```

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "neo-scraper", "name": "neo-scraper",
"version": "0.10.0", "version": "0.10.1",
"description": "Scrape a booru-like post object from the DOM.", "description": "Scrape a booru-like post object from the DOM.",
"type": "module", "type": "module",
"main": "dist/index.js", "main": "dist/index.js",

4
src/ScrapeEngine.ts
View File

@ -32,7 +32,7 @@ export class ScrapedTag {
} }
export class ScrapedNote { export class ScrapedNote {
constructor(public text: string, public polygon: number[][]) { } constructor(public text: string, public polygon: number[][]) {}
} }
export class ScrapeResult { export class ScrapeResult {
@ -95,7 +95,7 @@ export abstract class ScrapeEngineBase implements ScrapeEngine {
abstract supportedHosts: string[]; abstract supportedHosts: string[];
canImport(url: Location): boolean { canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host) != -1; return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
} }
abstract scrapeDocument(document: Document): ScrapeResult; abstract scrapeDocument(document: Document): ScrapeResult;

2
src/engines/Danbooru.ts
View File

@ -6,7 +6,7 @@ export default class Danbooru extends ScrapeEngineBase {
name = "danbooru"; name = "danbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = []; notes = [];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"]; supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

6
src/engines/FurAffinity.ts
View File

@ -5,7 +5,7 @@ export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity"; name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = []; notes = [];
supportedHosts = ["www.furaffinity.net"]; supportedHosts = ["furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);
@ -47,12 +47,12 @@ export default class FurAffinity extends ScrapeEngineBase {
} }
// Set tags // Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags")) post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
.map((x) => (x as HTMLSpanElement).innerText) .map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x)); .map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist // Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText; const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
if (submissionUser) { if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist")); post.tags.push(new ScrapedTag(submissionUser, "artist"));
} }

4
src/engines/Gelbooru.test.ts
View File

@ -29,7 +29,7 @@ test("get gelbooru.com post 4925055", async () => {
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958"); expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy"); expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
}); });
test("get gelbooru.com post 6225495 with notes", async () => { test("get gelbooru.com post 6225495 with notes", async () => {
@ -50,7 +50,7 @@ test("get gelbooru.com post 6225495 with notes", async () => {
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816"); expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy"); expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png"); expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
}); });
test("get rule34.xxx post 2740994", async () => { test("get rule34.xxx post 2740994", async () => {

12
src/engines/Gelbooru.ts
View File

@ -13,7 +13,15 @@ export default class Gelbooru extends ScrapeEngineBase {
name = "gelbooru"; name = "gelbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = ["Supported features might vary between hosts."]; notes = ["Supported features might vary between hosts."];
supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com"]; supportedHosts = [
"safebooru.org",
"gelbooru.com",
"rule34.xxx",
"tbib.org",
"xbooru.com",
"mspabooru.com",
"hypnohub.net",
];
private readonly classNameToCategoryMap: CategoryMap = { private readonly classNameToCategoryMap: CategoryMap = {
"tag-type-copyright": "copyright", "tag-type-copyright": "copyright",
@ -30,6 +38,8 @@ export default class Gelbooru extends ScrapeEngineBase {
case "rule34.xxx": case "rule34.xxx":
case "tbib.org": case "tbib.org":
case "xbooru.com": case "xbooru.com":
case "mspabooru.com":
case "hypnohub.net":
version = Version.v020; version = Version.v020;
break; break;
case "gelbooru.com": case "gelbooru.com":

2
src/engines/Moebooru.ts
View File

@ -6,7 +6,7 @@ export default class Moebooru extends ScrapeEngineBase {
name = "moebooru"; name = "moebooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = []; notes = [];
supportedHosts = ["yande.re", "konachan.com"]; supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

2
src/engines/Pixiv.ts
View File

@ -25,7 +25,7 @@ export default class Pixiv extends ScrapeEngineBase {
name = "pixiv"; name = "pixiv";
features: ScrapeEngineFeature[] = ["content"]; features: ScrapeEngineFeature[] = ["content"];
notes = []; notes = [];
supportedHosts = ["pixiv.net", "www.pixiv.net"]; supportedHosts = ["pixiv.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

2
src/engines/Zerochan.ts
View File

@ -6,7 +6,7 @@ export default class Zerochan extends ScrapeEngineBase {
name = "zerochan"; name = "zerochan";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"]; features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."]; notes = ["Rating is assumed to be safe."];
supportedHosts = ["www.zerochan.net"]; supportedHosts = ["zerochan.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);