diff --git a/README.md b/README.md index b06cc4b..8e4a22c 100644 --- a/README.md +++ b/README.md @@ -4,25 +4,28 @@ Scrape a booru-like post object from the DOM. ## Supported sites -| Engine | Sites | Features | Notes | -| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | -| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | | -| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | -| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | -| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | -| inkbunny | inkbunny.net | content, rating, resolution, tags | | -| moebooru | yande.re, konachan.com, konachan.net, www.sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | | -| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | | -| pixiv | pixiv.net, www.pixiv.net | content | | -| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | -| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | -| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | -| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | -| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | -| twitter | twitter.com, mobile.twitter.com | content | | -| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | -| fallback | | content | Tries to find the largest image or video on the current page. | +| Engine | Sites | Features | Notes | +|---|---|---|---|---| +| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| bluesky | bsky.app | content | | +| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | +| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | +| E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. | +| furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | | +| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | +| inkbunny | inkbunny.net | content, rating, resolution, tags | | +| moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | | +| nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. | +| Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | | +| pixiv | pixiv.net | content | | +| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | +| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | +| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | +| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | +| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | +| twitter | twitter.com, mobile.twitter.com, x.com | content | | +| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| fallback | | content | Tries to find the largest image or video on the current page. | ## Development @@ -34,5 +37,6 @@ pnpm install pnpm build # or pnpm watch in another terminal pnpm link --global cd ../SzuruChrome +pnpm link --global neo-scraper pnpm dev ``` diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index 97eb453..9dcc890 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -32,7 +32,7 @@ export class ScrapedTag { } export class ScrapedNote { - constructor(public text: string, public polygon: number[][]) { } + constructor(public text: string, public polygon: number[][]) {} } export class ScrapeResult { @@ -95,7 +95,7 @@ export abstract class ScrapeEngineBase implements ScrapeEngine { abstract supportedHosts: string[]; canImport(url: Location): boolean { - return this.supportedHosts.indexOf(url.host) != -1; + return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1; } abstract scrapeDocument(document: Document): ScrapeResult; diff --git a/src/engines/Danbooru.ts b/src/engines/Danbooru.ts index 33780fe..a4ed604 100644 --- a/src/engines/Danbooru.ts +++ b/src/engines/Danbooru.ts @@ -6,7 +6,7 @@ export default class Danbooru extends ScrapeEngineBase { name = "danbooru"; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; notes = []; - supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us", "hypnohub.net"]; + supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us"]; scrapeDocument(document: Document): ScrapeResult { const result = new ScrapeResult(this.name); diff --git a/src/engines/FurAffinity.ts b/src/engines/FurAffinity.ts index 56abd0d..34a9d33 100644 --- a/src/engines/FurAffinity.ts +++ b/src/engines/FurAffinity.ts @@ -1,64 +1,64 @@ -import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; -import { guessContentType, parseResolutionString } from "../Utility.js"; - -export default class FurAffinity extends ScrapeEngineBase { - name = "furaffinity"; - features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"]; - notes = []; - supportedHosts = ["www.furaffinity.net"]; - - scrapeDocument(document: Document): ScrapeResult { - const result = new ScrapeResult(this.name); - const post = new ScrapedPost(); - post.pageUrl = document.location.href; - - // Set image url - // Could use the download button here, but as far as I can tell it doesn't matter. - const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement; - post.contentUrl = downloadEl?.src; - - // Set content type - post.contentType = guessContentType(post.contentUrl); - - // Set resolution - const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []); - const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size"); - if (sizeDiv) { - const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText); - post.resolution = resolution; - } - - // Set rating - const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map( - (x) => x as HTMLSpanElement - ); - - if (ratingEls.length > 0) - switch (ratingEls[0].innerText) { - case "General": - post.rating = "safe"; - break; - case "Mature": - post.rating = "sketchy"; - break; - case "Adult": - post.rating = "unsafe"; - break; - } - - // Set tags - post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags")) - .map((x) => (x as HTMLSpanElement).innerText) - .map((x) => new ScrapedTag(x)); - - // Set user who submitted the art as the artist - const submissionUser = document.querySelector(".submission-id-sub-container > a > strong")?.innerText; - if (submissionUser) { - post.tags.push(new ScrapedTag(submissionUser, "artist")); - } - - result.tryAddPost(post); - - return result; - } -} +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; +import { guessContentType, parseResolutionString } from "../Utility.js"; + +export default class FurAffinity extends ScrapeEngineBase { + name = "furaffinity"; + features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"]; + notes = []; + supportedHosts = ["furaffinity.net"]; + + scrapeDocument(document: Document): ScrapeResult { + const result = new ScrapeResult(this.name); + const post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + // Could use the download button here, but as far as I can tell it doesn't matter. + const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement; + post.contentUrl = downloadEl?.src; + + // Set content type + post.contentType = guessContentType(post.contentUrl); + + // Set resolution + const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []); + const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size"); + if (sizeDiv) { + const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText); + post.resolution = resolution; + } + + // Set rating + const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map( + (x) => x as HTMLSpanElement + ); + + if (ratingEls.length > 0) + switch (ratingEls[0].innerText) { + case "General": + post.rating = "safe"; + break; + case "Mature": + post.rating = "sketchy"; + break; + case "Adult": + post.rating = "unsafe"; + break; + } + + // Set tags + post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags")) + .map((x) => (x as HTMLSpanElement).innerText) + .map((x) => new ScrapedTag(x)); + + // Set user who submitted the art as the artist + const submissionUser = document.querySelector(".submission-id-sub-container > a > strong")?.innerText; + if (submissionUser) { + post.tags.push(new ScrapedTag(submissionUser, "artist")); + } + + result.tryAddPost(post); + + return result; + } +} diff --git a/src/engines/Gelbooru.ts b/src/engines/Gelbooru.ts index 620cb14..cca525e 100644 --- a/src/engines/Gelbooru.ts +++ b/src/engines/Gelbooru.ts @@ -13,7 +13,15 @@ export default class Gelbooru extends ScrapeEngineBase { name = "gelbooru"; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; notes = ["Supported features might vary between hosts."]; - supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com", "mspabooru.com"]; + supportedHosts = [ + "safebooru.org", + "gelbooru.com", + "rule34.xxx", + "tbib.org", + "xbooru.com", + "mspabooru.com", + "hypnohub.net", + ]; private readonly classNameToCategoryMap: CategoryMap = { "tag-type-copyright": "copyright", @@ -31,6 +39,7 @@ export default class Gelbooru extends ScrapeEngineBase { case "tbib.org": case "xbooru.com": case "mspabooru.com": + case "hypnohub.net": version = Version.v020; break; case "gelbooru.com": diff --git a/src/engines/Moebooru.ts b/src/engines/Moebooru.ts index 5021cb8..2a24470 100644 --- a/src/engines/Moebooru.ts +++ b/src/engines/Moebooru.ts @@ -6,7 +6,7 @@ export default class Moebooru extends ScrapeEngineBase { name = "moebooru"; features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; notes = []; - supportedHosts = ["yande.re", "konachan.com", "konachan.net", "www.sakugabooru.com", "img.genshiken-itb.org"]; + supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"]; scrapeDocument(document: Document): ScrapeResult { const result = new ScrapeResult(this.name); diff --git a/src/engines/Pixiv.ts b/src/engines/Pixiv.ts index ba1ab91..afdac74 100644 --- a/src/engines/Pixiv.ts +++ b/src/engines/Pixiv.ts @@ -25,7 +25,7 @@ export default class Pixiv extends ScrapeEngineBase { name = "pixiv"; features: ScrapeEngineFeature[] = ["content"]; notes = []; - supportedHosts = ["pixiv.net", "www.pixiv.net"]; + supportedHosts = ["pixiv.net"]; scrapeDocument(document: Document): ScrapeResult { const result = new ScrapeResult(this.name); diff --git a/src/engines/Zerochan.ts b/src/engines/Zerochan.ts index 2289d29..f9f631e 100644 --- a/src/engines/Zerochan.ts +++ b/src/engines/Zerochan.ts @@ -6,7 +6,7 @@ export default class Zerochan extends ScrapeEngineBase { name = "zerochan"; features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"]; notes = ["Rating is assumed to be safe."]; - supportedHosts = ["www.zerochan.net"]; + supportedHosts = ["zerochan.net"]; scrapeDocument(document: Document): ScrapeResult { const result = new ScrapeResult(this.name);