Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix some things

pull/20/head
neobooru 2025-04-12 00:10:28 +02:00
parent 832b3d286d
commit 4692fe757d
8 changed files with 103 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
README.md
View File

@ -4,25 +4,28 @@ Scrape a booru-like post object from the DOM.
## Supported sites ## Supported sites
| Engine | Sites | Features | Notes | | Engine | Sites | Features | Notes |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | |---|---|---|---|---|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | | animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | | | bluesky | bsky.app | content | |
| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | | danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | | e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | | E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | | | furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | |
| moebooru | yande.re, konachan.com, konachan.net, www.sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | | | gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | | | inkbunny | inkbunny.net | content, rating, resolution, tags | |
| pixiv | pixiv.net, www.pixiv.net | content | | | moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | | nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | | Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | | pixiv | pixiv.net | content | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | | reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | | rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| twitter | twitter.com, mobile.twitter.com | content | | | sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | | shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| fallback | | content | Tries to find the largest image or video on the current page. | | shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com, x.com | content | |
| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |
## Development ## Development
@ -34,5 +37,6 @@ pnpm install
pnpm build # or pnpm watch in another terminal pnpm build # or pnpm watch in another terminal
pnpm link --global pnpm link --global
cd ../SzuruChrome cd ../SzuruChrome
pnpm link --global neo-scraper
pnpm dev pnpm dev
``` ```

4
src/ScrapeEngine.ts
View File

@ -32,7 +32,7 @@ export class ScrapedTag {
} }
export class ScrapedNote { export class ScrapedNote {
constructor(public text: string, public polygon: number[][]) { } constructor(public text: string, public polygon: number[][]) {}
} }
export class ScrapeResult { export class ScrapeResult {
@ -95,7 +95,7 @@ export abstract class ScrapeEngineBase implements ScrapeEngine {
abstract supportedHosts: string[]; abstract supportedHosts: string[];
canImport(url: Location): boolean { canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host) != -1; return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
} }
abstract scrapeDocument(document: Document): ScrapeResult; abstract scrapeDocument(document: Document): ScrapeResult;

2
src/engines/Danbooru.ts
View File

@ -6,7 +6,7 @@ export default class Danbooru extends ScrapeEngineBase {
name = "danbooru"; name = "danbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = []; notes = [];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us", "hypnohub.net"]; supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

2
src/engines/FurAffinity.ts
View File

@ -5,7 +5,7 @@ export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity"; name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = []; notes = [];
supportedHosts = ["www.furaffinity.net"]; supportedHosts = ["furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

11
src/engines/Gelbooru.ts
View File

@ -13,7 +13,15 @@ export default class Gelbooru extends ScrapeEngineBase {
name = "gelbooru"; name = "gelbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = ["Supported features might vary between hosts."]; notes = ["Supported features might vary between hosts."];
supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com", "mspabooru.com"]; supportedHosts = [
"safebooru.org",
"gelbooru.com",
"rule34.xxx",
"tbib.org",
"xbooru.com",
"mspabooru.com",
"hypnohub.net",
];
private readonly classNameToCategoryMap: CategoryMap = { private readonly classNameToCategoryMap: CategoryMap = {
"tag-type-copyright": "copyright", "tag-type-copyright": "copyright",
@ -31,6 +39,7 @@ export default class Gelbooru extends ScrapeEngineBase {
case "tbib.org": case "tbib.org":
case "xbooru.com": case "xbooru.com":
case "mspabooru.com": case "mspabooru.com":
case "hypnohub.net":
version = Version.v020; version = Version.v020;
break; break;
case "gelbooru.com": case "gelbooru.com":

2
src/engines/Moebooru.ts
View File

@ -6,7 +6,7 @@ export default class Moebooru extends ScrapeEngineBase {
name = "moebooru"; name = "moebooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = []; notes = [];
supportedHosts = ["yande.re", "konachan.com", "konachan.net", "www.sakugabooru.com", "img.genshiken-itb.org"]; supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

2
src/engines/Pixiv.ts
View File

@ -25,7 +25,7 @@ export default class Pixiv extends ScrapeEngineBase {
name = "pixiv"; name = "pixiv";
features: ScrapeEngineFeature[] = ["content"]; features: ScrapeEngineFeature[] = ["content"];
notes = []; notes = [];
supportedHosts = ["pixiv.net", "www.pixiv.net"]; supportedHosts = ["pixiv.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);

2
src/engines/Zerochan.ts
View File

@ -6,7 +6,7 @@ export default class Zerochan extends ScrapeEngineBase {
name = "zerochan"; name = "zerochan";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"]; features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."]; notes = ["Rating is assumed to be safe."];
supportedHosts = ["www.zerochan.net"]; supportedHosts = ["zerochan.net"];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); const result = new ScrapeResult(this.name);