Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix + bump version

pull/20/head
neobooru 2025-04-12 01:03:38 +02:00
parent 4692fe757d
commit 51be6dfdcd
3 changed files with 87 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "neo-scraper",
"version": "0.10.0",
"version": "0.10.1",
"description": "Scrape a booru-like post object from the DOM.",
"type": "module",
"main": "dist/index.js",

4
src/engines/FurAffinity.ts
View File

@ -47,12 +47,12 @@ export default class FurAffinity extends ScrapeEngineBase {
}
// Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
.map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText;
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist"));
}

168
src/engines/Gelbooru.test.ts
View File

@ -1,84 +1,84 @@
import { scrapeUrl } from "../test/engine.js";
test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
});
test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("maosen", "artist");
expect(res.posts[0]).toHaveTag("butterfly");
expect(res.posts[0]).toHaveResolution([720, 1103]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
test("get gelbooru.com post 6225495 with notes", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("teeth");
expect(res.posts[0]).toHaveTag("highres");
expect(res.posts[0]).toHaveTag("nintendo");
expect(res.posts[0]).toHaveTag("link");
expect(res.posts[0]).toHaveTag("ttanuu.");
expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
expect(res.posts[0]).toHaveNote("But no need to worry!");
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
expect(res.posts[0]).toHaveResolution([1600, 2200]);
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
});
test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});
test("get xbooru.com post 834083", async () => {
// Don't worry this isn't a cursed image.
const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
expect(res.posts[0]).toHaveTag("nurse");
expect(res.posts[0]).toHaveTag("standing");
expect(res.posts[0]).toHaveResolution([1024, 1878]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
});
import { scrapeUrl } from "../test/engine.js";
test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
});
test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("maosen", "artist");
expect(res.posts[0]).toHaveTag("butterfly");
expect(res.posts[0]).toHaveResolution([720, 1103]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
test("get gelbooru.com post 6225495 with notes", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("teeth");
expect(res.posts[0]).toHaveTag("highres");
expect(res.posts[0]).toHaveTag("nintendo");
expect(res.posts[0]).toHaveTag("link");
expect(res.posts[0]).toHaveTag("ttanuu.");
expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
expect(res.posts[0]).toHaveNote("But no need to worry!");
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
expect(res.posts[0]).toHaveResolution([1600, 2200]);
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
});
test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});
test("get xbooru.com post 834083", async () => {
// Don't worry this isn't a cursed image.
const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
expect(res.posts[0]).toHaveTag("nurse");
expect(res.posts[0]).toHaveTag("standing");
expect(res.posts[0]).toHaveResolution([1024, 1878]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
});