pull/19/head
commit
bf9f3b8e7a
42
README.md
42
README.md
|
|
@ -4,25 +4,28 @@ Scrape a booru-like post object from the DOM.
|
||||||
|
|
||||||
## Supported sites
|
## Supported sites
|
||||||
|
|
||||||
| Engine | Sites | Features | Notes |
|
| Engine | Sites | Features | Notes |
|
||||||
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
|
|---|---|---|---|---|
|
||||||
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
|
||||||
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
|
| bluesky | bsky.app | content | |
|
||||||
| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | |
|
| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
|
||||||
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | |
|
| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | |
|
||||||
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
|
| E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
|
||||||
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
|
| furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | |
|
||||||
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | |
|
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
|
||||||
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
|
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
|
||||||
| pixiv | pixiv.net, www.pixiv.net | content | |
|
| moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
|
||||||
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
|
| nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
|
||||||
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
|
| Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
|
||||||
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
|
| pixiv | pixiv.net | content | |
|
||||||
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
|
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
|
||||||
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
|
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
|
||||||
| twitter | twitter.com, mobile.twitter.com | content | |
|
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
|
||||||
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
|
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
|
||||||
| fallback | | content | Tries to find the largest image or video on the current page. |
|
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
|
||||||
|
| twitter | twitter.com, mobile.twitter.com, x.com | content | |
|
||||||
|
| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
|
||||||
|
| fallback | | content | Tries to find the largest image or video on the current page. |
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
|
|
@ -34,5 +37,6 @@ pnpm install
|
||||||
pnpm build # or pnpm watch in another terminal
|
pnpm build # or pnpm watch in another terminal
|
||||||
pnpm link --global
|
pnpm link --global
|
||||||
cd ../SzuruChrome
|
cd ../SzuruChrome
|
||||||
|
pnpm link --global neo-scraper
|
||||||
pnpm dev
|
pnpm dev
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "neo-scraper",
|
"name": "neo-scraper",
|
||||||
"version": "0.10.0",
|
"version": "0.10.1",
|
||||||
"description": "Scrape a booru-like post object from the DOM.",
|
"description": "Scrape a booru-like post object from the DOM.",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ export class ScrapedTag {
|
||||||
}
|
}
|
||||||
|
|
||||||
export class ScrapedNote {
|
export class ScrapedNote {
|
||||||
constructor(public text: string, public polygon: number[][]) { }
|
constructor(public text: string, public polygon: number[][]) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
export class ScrapeResult {
|
export class ScrapeResult {
|
||||||
|
|
@ -95,7 +95,7 @@ export abstract class ScrapeEngineBase implements ScrapeEngine {
|
||||||
abstract supportedHosts: string[];
|
abstract supportedHosts: string[];
|
||||||
|
|
||||||
canImport(url: Location): boolean {
|
canImport(url: Location): boolean {
|
||||||
return this.supportedHosts.indexOf(url.host) != -1;
|
return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
abstract scrapeDocument(document: Document): ScrapeResult;
|
abstract scrapeDocument(document: Document): ScrapeResult;
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ export default class Danbooru extends ScrapeEngineBase {
|
||||||
name = "danbooru";
|
name = "danbooru";
|
||||||
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
|
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
|
||||||
notes = [];
|
notes = [];
|
||||||
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"];
|
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us", "sonohara.donmai.us", "hijiribe.donmai.us"];
|
||||||
|
|
||||||
scrapeDocument(document: Document): ScrapeResult {
|
scrapeDocument(document: Document): ScrapeResult {
|
||||||
const result = new ScrapeResult(this.name);
|
const result = new ScrapeResult(this.name);
|
||||||
|
|
|
||||||
|
|
@ -1,64 +1,64 @@
|
||||||
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
|
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
|
||||||
import { guessContentType, parseResolutionString } from "../Utility.js";
|
import { guessContentType, parseResolutionString } from "../Utility.js";
|
||||||
|
|
||||||
export default class FurAffinity extends ScrapeEngineBase {
|
export default class FurAffinity extends ScrapeEngineBase {
|
||||||
name = "furaffinity";
|
name = "furaffinity";
|
||||||
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
|
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
|
||||||
notes = [];
|
notes = [];
|
||||||
supportedHosts = ["www.furaffinity.net"];
|
supportedHosts = ["furaffinity.net"];
|
||||||
|
|
||||||
scrapeDocument(document: Document): ScrapeResult {
|
scrapeDocument(document: Document): ScrapeResult {
|
||||||
const result = new ScrapeResult(this.name);
|
const result = new ScrapeResult(this.name);
|
||||||
const post = new ScrapedPost();
|
const post = new ScrapedPost();
|
||||||
post.pageUrl = document.location.href;
|
post.pageUrl = document.location.href;
|
||||||
|
|
||||||
// Set image url
|
// Set image url
|
||||||
// Could use the download button here, but as far as I can tell it doesn't matter.
|
// Could use the download button here, but as far as I can tell it doesn't matter.
|
||||||
const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement;
|
const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement;
|
||||||
post.contentUrl = downloadEl?.src;
|
post.contentUrl = downloadEl?.src;
|
||||||
|
|
||||||
// Set content type
|
// Set content type
|
||||||
post.contentType = guessContentType(post.contentUrl);
|
post.contentType = guessContentType(post.contentUrl);
|
||||||
|
|
||||||
// Set resolution
|
// Set resolution
|
||||||
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
|
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
|
||||||
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
|
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
|
||||||
if (sizeDiv) {
|
if (sizeDiv) {
|
||||||
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
|
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
|
||||||
post.resolution = resolution;
|
post.resolution = resolution;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set rating
|
// Set rating
|
||||||
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
|
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
|
||||||
(x) => x as HTMLSpanElement
|
(x) => x as HTMLSpanElement
|
||||||
);
|
);
|
||||||
|
|
||||||
if (ratingEls.length > 0)
|
if (ratingEls.length > 0)
|
||||||
switch (ratingEls[0].innerText) {
|
switch (ratingEls[0].innerText) {
|
||||||
case "General":
|
case "General":
|
||||||
post.rating = "safe";
|
post.rating = "safe";
|
||||||
break;
|
break;
|
||||||
case "Mature":
|
case "Mature":
|
||||||
post.rating = "sketchy";
|
post.rating = "sketchy";
|
||||||
break;
|
break;
|
||||||
case "Adult":
|
case "Adult":
|
||||||
post.rating = "unsafe";
|
post.rating = "unsafe";
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set tags
|
// Set tags
|
||||||
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
|
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
|
||||||
.map((x) => (x as HTMLSpanElement).innerText)
|
.map((x) => (x as HTMLSpanElement).innerText)
|
||||||
.map((x) => new ScrapedTag(x));
|
.map((x) => new ScrapedTag(x));
|
||||||
|
|
||||||
// Set user who submitted the art as the artist
|
// Set user who submitted the art as the artist
|
||||||
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText;
|
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
|
||||||
if (submissionUser) {
|
if (submissionUser) {
|
||||||
post.tags.push(new ScrapedTag(submissionUser, "artist"));
|
post.tags.push(new ScrapedTag(submissionUser, "artist"));
|
||||||
}
|
}
|
||||||
|
|
||||||
result.tryAddPost(post);
|
result.tryAddPost(post);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,84 +1,84 @@
|
||||||
import { scrapeUrl } from "../test/engine.js";
|
import { scrapeUrl } from "../test/engine.js";
|
||||||
|
|
||||||
test("get safebooru.org post 2871532", async () => {
|
test("get safebooru.org post 2871532", async () => {
|
||||||
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
|
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
|
||||||
const res = await scrapeUrl(pageUrl);
|
const res = await scrapeUrl(pageUrl);
|
||||||
expect(res.posts.length).toBe(1);
|
expect(res.posts.length).toBe(1);
|
||||||
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
|
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
|
||||||
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
|
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
|
||||||
expect(res.posts[0]).toHaveTag("bangs");
|
expect(res.posts[0]).toHaveTag("bangs");
|
||||||
expect(res.posts[0]).toHaveTag("shirt");
|
expect(res.posts[0]).toHaveTag("shirt");
|
||||||
expect(res.posts[0]).toHaveTag("skirt");
|
expect(res.posts[0]).toHaveTag("skirt");
|
||||||
expect(res.posts[0]).toHaveTag("t-shirt");
|
expect(res.posts[0]).toHaveTag("t-shirt");
|
||||||
expect(res.posts[0]).toHaveTag("long_hair");
|
expect(res.posts[0]).toHaveTag("long_hair");
|
||||||
expect(res.posts[0]).toHaveResolution([1142, 1600]);
|
expect(res.posts[0]).toHaveResolution([1142, 1600]);
|
||||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||||
expect(res.posts[0].rating).toBe("safe");
|
expect(res.posts[0].rating).toBe("safe");
|
||||||
expect(res.posts[0].contentUrl).toContain(
|
expect(res.posts[0].contentUrl).toContain(
|
||||||
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
|
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("get gelbooru.com post 4925055", async () => {
|
test("get gelbooru.com post 4925055", async () => {
|
||||||
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
|
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
|
||||||
const res = await scrapeUrl(pageUrl);
|
const res = await scrapeUrl(pageUrl);
|
||||||
expect(res.posts.length).toBe(1);
|
expect(res.posts.length).toBe(1);
|
||||||
expect(res.posts[0]).toHaveTag("maosen", "artist");
|
expect(res.posts[0]).toHaveTag("maosen", "artist");
|
||||||
expect(res.posts[0]).toHaveTag("butterfly");
|
expect(res.posts[0]).toHaveTag("butterfly");
|
||||||
expect(res.posts[0]).toHaveResolution([720, 1103]);
|
expect(res.posts[0]).toHaveResolution([720, 1103]);
|
||||||
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
|
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
|
||||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||||
expect(res.posts[0].rating).toBe("sketchy");
|
expect(res.posts[0].rating).toBe("sketchy");
|
||||||
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
|
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("get gelbooru.com post 6225495 with notes", async () => {
|
test("get gelbooru.com post 6225495 with notes", async () => {
|
||||||
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
|
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
|
||||||
const res = await scrapeUrl(pageUrl);
|
const res = await scrapeUrl(pageUrl);
|
||||||
expect(res.posts.length).toBe(1);
|
expect(res.posts.length).toBe(1);
|
||||||
expect(res.posts[0]).toHaveTag("hair_up");
|
expect(res.posts[0]).toHaveTag("hair_up");
|
||||||
expect(res.posts[0]).toHaveTag("hair_up");
|
expect(res.posts[0]).toHaveTag("hair_up");
|
||||||
expect(res.posts[0]).toHaveTag("teeth");
|
expect(res.posts[0]).toHaveTag("teeth");
|
||||||
expect(res.posts[0]).toHaveTag("highres");
|
expect(res.posts[0]).toHaveTag("highres");
|
||||||
expect(res.posts[0]).toHaveTag("nintendo");
|
expect(res.posts[0]).toHaveTag("nintendo");
|
||||||
expect(res.posts[0]).toHaveTag("link");
|
expect(res.posts[0]).toHaveTag("link");
|
||||||
expect(res.posts[0]).toHaveTag("ttanuu.");
|
expect(res.posts[0]).toHaveTag("ttanuu.");
|
||||||
expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
|
expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
|
||||||
expect(res.posts[0]).toHaveNote("But no need to worry!");
|
expect(res.posts[0]).toHaveNote("But no need to worry!");
|
||||||
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
|
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
|
||||||
expect(res.posts[0]).toHaveResolution([1600, 2200]);
|
expect(res.posts[0]).toHaveResolution([1600, 2200]);
|
||||||
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
|
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
|
||||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||||
expect(res.posts[0].rating).toBe("sketchy");
|
expect(res.posts[0].rating).toBe("sketchy");
|
||||||
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
|
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("get rule34.xxx post 2740994", async () => {
|
test("get rule34.xxx post 2740994", async () => {
|
||||||
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
|
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
|
||||||
const res = await scrapeUrl(pageUrl);
|
const res = await scrapeUrl(pageUrl);
|
||||||
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
|
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
|
||||||
expect(res.posts[0]).toHaveTag("nintendo", "copyright");
|
expect(res.posts[0]).toHaveTag("nintendo", "copyright");
|
||||||
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
|
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
|
||||||
expect(res.posts[0]).toHaveTag("blue_hair");
|
expect(res.posts[0]).toHaveTag("blue_hair");
|
||||||
expect(res.posts[0]).toHaveTag("solo");
|
expect(res.posts[0]).toHaveTag("solo");
|
||||||
expect(res.posts[0]).toHaveResolution([600, 800]);
|
expect(res.posts[0]).toHaveResolution([600, 800]);
|
||||||
expect(res.posts.length).toBe(1);
|
expect(res.posts.length).toBe(1);
|
||||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||||
expect(res.posts[0].rating).toBe("safe");
|
expect(res.posts[0].rating).toBe("safe");
|
||||||
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
|
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("get xbooru.com post 834083", async () => {
|
test("get xbooru.com post 834083", async () => {
|
||||||
// Don't worry this isn't a cursed image.
|
// Don't worry this isn't a cursed image.
|
||||||
const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
|
const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
|
||||||
const res = await scrapeUrl(pageUrl);
|
const res = await scrapeUrl(pageUrl);
|
||||||
expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
|
expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
|
||||||
expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
|
expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
|
||||||
expect(res.posts[0]).toHaveTag("nurse");
|
expect(res.posts[0]).toHaveTag("nurse");
|
||||||
expect(res.posts[0]).toHaveTag("standing");
|
expect(res.posts[0]).toHaveTag("standing");
|
||||||
expect(res.posts[0]).toHaveResolution([1024, 1878]);
|
expect(res.posts[0]).toHaveResolution([1024, 1878]);
|
||||||
expect(res.posts.length).toBe(1);
|
expect(res.posts.length).toBe(1);
|
||||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||||
expect(res.posts[0].rating).toBe("safe");
|
expect(res.posts[0].rating).toBe("safe");
|
||||||
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
|
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,15 @@ export default class Gelbooru extends ScrapeEngineBase {
|
||||||
name = "gelbooru";
|
name = "gelbooru";
|
||||||
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
|
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
|
||||||
notes = ["Supported features might vary between hosts."];
|
notes = ["Supported features might vary between hosts."];
|
||||||
supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com"];
|
supportedHosts = [
|
||||||
|
"safebooru.org",
|
||||||
|
"gelbooru.com",
|
||||||
|
"rule34.xxx",
|
||||||
|
"tbib.org",
|
||||||
|
"xbooru.com",
|
||||||
|
"mspabooru.com",
|
||||||
|
"hypnohub.net",
|
||||||
|
];
|
||||||
|
|
||||||
private readonly classNameToCategoryMap: CategoryMap = {
|
private readonly classNameToCategoryMap: CategoryMap = {
|
||||||
"tag-type-copyright": "copyright",
|
"tag-type-copyright": "copyright",
|
||||||
|
|
@ -30,6 +38,8 @@ export default class Gelbooru extends ScrapeEngineBase {
|
||||||
case "rule34.xxx":
|
case "rule34.xxx":
|
||||||
case "tbib.org":
|
case "tbib.org":
|
||||||
case "xbooru.com":
|
case "xbooru.com":
|
||||||
|
case "mspabooru.com":
|
||||||
|
case "hypnohub.net":
|
||||||
version = Version.v020;
|
version = Version.v020;
|
||||||
break;
|
break;
|
||||||
case "gelbooru.com":
|
case "gelbooru.com":
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ export default class Moebooru extends ScrapeEngineBase {
|
||||||
name = "moebooru";
|
name = "moebooru";
|
||||||
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
|
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
|
||||||
notes = [];
|
notes = [];
|
||||||
supportedHosts = ["yande.re", "konachan.com"];
|
supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];
|
||||||
|
|
||||||
scrapeDocument(document: Document): ScrapeResult {
|
scrapeDocument(document: Document): ScrapeResult {
|
||||||
const result = new ScrapeResult(this.name);
|
const result = new ScrapeResult(this.name);
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ export default class Pixiv extends ScrapeEngineBase {
|
||||||
name = "pixiv";
|
name = "pixiv";
|
||||||
features: ScrapeEngineFeature[] = ["content"];
|
features: ScrapeEngineFeature[] = ["content"];
|
||||||
notes = [];
|
notes = [];
|
||||||
supportedHosts = ["pixiv.net", "www.pixiv.net"];
|
supportedHosts = ["pixiv.net"];
|
||||||
|
|
||||||
scrapeDocument(document: Document): ScrapeResult {
|
scrapeDocument(document: Document): ScrapeResult {
|
||||||
const result = new ScrapeResult(this.name);
|
const result = new ScrapeResult(this.name);
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ export default class Zerochan extends ScrapeEngineBase {
|
||||||
name = "zerochan";
|
name = "zerochan";
|
||||||
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
|
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
|
||||||
notes = ["Rating is assumed to be safe."];
|
notes = ["Rating is assumed to be safe."];
|
||||||
supportedHosts = ["www.zerochan.net"];
|
supportedHosts = ["zerochan.net"];
|
||||||
|
|
||||||
scrapeDocument(document: Document): ScrapeResult {
|
scrapeDocument(document: Document): ScrapeResult {
|
||||||
const result = new ScrapeResult(this.name);
|
const result = new ScrapeResult(this.name);
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue