Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix some things

pull/20/head
neobooru 2025-04-12 00:10:28 +02:00
parent 832b3d286d
commit 4692fe757d
8 changed files with 103 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
README.md
View File

@ -4,25 +4,28 @@ Scrape a booru-like post object from the DOM.
## Supported sites
| Engine | Sites | Features | Notes |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | |
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com, konachan.net, www.sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net, www.pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com | content | |
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |
| Engine | Sites | Features | Notes |
|---|---|---|---|---|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| bluesky | bsky.app | content | |
| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | |
| E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
| furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
| nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
| Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com, x.com | content | |
| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |
## Development
@ -34,5 +37,6 @@ pnpm install
pnpm build # or pnpm watch in another terminal
pnpm link --global
cd ../SzuruChrome
pnpm link --global neo-scraper
pnpm dev
```

4
src/ScrapeEngine.ts
View File

@ -32,7 +32,7 @@ export class ScrapedTag {
}
export class ScrapedNote {
constructor(public text: string, public polygon: number[][]) { }
constructor(public text: string, public polygon: number[][]) {}
}
export class ScrapeResult {
@ -95,7 +95,7 @@ export abstract class ScrapeEngineBase implements ScrapeEngine {
abstract supportedHosts: string[];
canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host) != -1;
return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
}
abstract scrapeDocument(document: Document): ScrapeResult;

2
src/engines/Danbooru.ts
View File

@ -6,7 +6,7 @@ export default class Danbooru extends ScrapeEngineBase {
name = "danbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = [];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us", "hypnohub.net"];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

128
src/engines/FurAffinity.ts
View File

@ -1,64 +1,64 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { guessContentType, parseResolutionString } from "../Utility.js";
export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = [];
supportedHosts = ["www.furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// Could use the download button here, but as far as I can tell it doesn't matter.
const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement;
post.contentUrl = downloadEl?.src;
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set resolution
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
if (sizeDiv) {
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
post.resolution = resolution;
}
// Set rating
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
(x) => x as HTMLSpanElement
);
if (ratingEls.length > 0)
switch (ratingEls[0].innerText) {
case "General":
post.rating = "safe";
break;
case "Mature":
post.rating = "sketchy";
break;
case "Adult":
post.rating = "unsafe";
break;
}
// Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
.map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText;
if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist"));
}
result.tryAddPost(post);
return result;
}
}
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { guessContentType, parseResolutionString } from "../Utility.js";
export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = [];
supportedHosts = ["furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// Could use the download button here, but as far as I can tell it doesn't matter.
const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement;
post.contentUrl = downloadEl?.src;
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set resolution
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
if (sizeDiv) {
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
post.resolution = resolution;
}
// Set rating
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
(x) => x as HTMLSpanElement
);
if (ratingEls.length > 0)
switch (ratingEls[0].innerText) {
case "General":
post.rating = "safe";
break;
case "Mature":
post.rating = "sketchy";
break;
case "Adult":
post.rating = "unsafe";
break;
}
// Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
.map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText;
if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist"));
}
result.tryAddPost(post);
return result;
}
}

11
src/engines/Gelbooru.ts
View File

@ -13,7 +13,15 @@ export default class Gelbooru extends ScrapeEngineBase {
name = "gelbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = ["Supported features might vary between hosts."];
supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com", "mspabooru.com"];
supportedHosts = [
"safebooru.org",
"gelbooru.com",
"rule34.xxx",
"tbib.org",
"xbooru.com",
"mspabooru.com",
"hypnohub.net",
];
private readonly classNameToCategoryMap: CategoryMap = {
"tag-type-copyright": "copyright",
@ -31,6 +39,7 @@ export default class Gelbooru extends ScrapeEngineBase {
case "tbib.org":
case "xbooru.com":
case "mspabooru.com":
case "hypnohub.net":
version = Version.v020;
break;
case "gelbooru.com":

2
src/engines/Moebooru.ts
View File

@ -6,7 +6,7 @@ export default class Moebooru extends ScrapeEngineBase {
name = "moebooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = [];
supportedHosts = ["yande.re", "konachan.com", "konachan.net", "www.sakugabooru.com", "img.genshiken-itb.org"];
supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

2
src/engines/Pixiv.ts
View File

@ -25,7 +25,7 @@ export default class Pixiv extends ScrapeEngineBase {
name = "pixiv";
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = ["pixiv.net", "www.pixiv.net"];
supportedHosts = ["pixiv.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);

2
src/engines/Zerochan.ts
View File

@ -6,7 +6,7 @@ export default class Zerochan extends ScrapeEngineBase {
name = "zerochan";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."];
supportedHosts = ["www.zerochan.net"];
supportedHosts = ["zerochan.net"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);