Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge https://github.com/neobooru/neo-scraper

pull/19/head
Kaias Blackwood 2025-08-14 17:58:51 +00:00
parent e1ffab613b 51be6dfdcd
commit bf9f3b8e7a
10 changed files with 189 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
README.md
View File

@ -5,23 +5,26 @@ Scrape a booru-like post object from the DOM.
## Supported sites
| Engine | Sites | Features | Notes |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
|---|---|---|---|---|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| bluesky | bsky.app | content | |
| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | |
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
| furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | |
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net, www.pixiv.net | content | |
| moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
| nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
| Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com | content | |
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com, x.com | content | |
| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |
## Development
@ -34,5 +37,6 @@ pnpm install
pnpm build # or pnpm watch in another terminal
pnpm link --global
cd ../SzuruChrome
pnpm link --global neo-scraper
pnpm dev
```

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "neo-scraper",
"version": "0.10.0",
"version": "0.10.1",
"description": "Scrape a booru-like post object from the DOM.",
"type": "module",
"main": "dist/index.js",

2
src/ScrapeEngine.ts
View File

@ -95,7 +95,7 @@ export abstract class ScrapeEngineBase implements ScrapeEngine {
abstract supportedHosts: string[];
canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host) != -1;
return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
}
abstract scrapeDocument(document: Document): ScrapeResult;

2
src/engines/Danbooru.ts
View File

@ -6,7 +6,7 @@ export default class Danbooru extends ScrapeEngineBase {
name = "danbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = [];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

6
src/engines/FurAffinity.ts
View File

@ -5,7 +5,7 @@ export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = [];
supportedHosts = ["www.furaffinity.net"];
supportedHosts = ["furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
@ -47,12 +47,12 @@ export default class FurAffinity extends ScrapeEngineBase {
}
// Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
.map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText;
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist"));
}

4
src/engines/Gelbooru.test.ts
View File

@ -29,7 +29,7 @@ test("get gelbooru.com post 4925055", async () => {
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
test("get gelbooru.com post 6225495 with notes", async () => {
@ -50,7 +50,7 @@ test("get gelbooru.com post 6225495 with notes", async () => {
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
});
test("get rule34.xxx post 2740994", async () => {

12
src/engines/Gelbooru.ts
View File

@ -13,7 +13,15 @@ export default class Gelbooru extends ScrapeEngineBase {
name = "gelbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = ["Supported features might vary between hosts."];
supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com"];
supportedHosts = [
"safebooru.org",
"gelbooru.com",
"rule34.xxx",
"tbib.org",
"xbooru.com",
"mspabooru.com",
"hypnohub.net",
];
private readonly classNameToCategoryMap: CategoryMap = {
"tag-type-copyright": "copyright",
@ -30,6 +38,8 @@ export default class Gelbooru extends ScrapeEngineBase {
case "rule34.xxx":
case "tbib.org":
case "xbooru.com":
case "mspabooru.com":
case "hypnohub.net":
version = Version.v020;
break;
case "gelbooru.com":

2
src/engines/Moebooru.ts
View File

@ -6,7 +6,7 @@ export default class Moebooru extends ScrapeEngineBase {
name = "moebooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = [];
supportedHosts = ["yande.re", "konachan.com"];
supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

2
src/engines/Pixiv.ts
View File

@ -25,7 +25,7 @@ export default class Pixiv extends ScrapeEngineBase {
name = "pixiv";
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = ["pixiv.net", "www.pixiv.net"];
supportedHosts = ["pixiv.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

2
src/engines/Zerochan.ts
View File

@ -6,7 +6,7 @@ export default class Zerochan extends ScrapeEngineBase {
name = "zerochan";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."];
supportedHosts = ["www.zerochan.net"];
supportedHosts = ["zerochan.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);