Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #15 from GlobalGit3/master 

Added Philomena Scraper
pull/16/head
Neo 2023-07-05 11:06:07 +00:00 committed by GitHub
parent 4254c5f5d8 5beea8cc69
commit c9daac63b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 176 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -5,7 +5,7 @@ Scrape a booru-like post object from the DOM.
## Supported sites
| Engine | Sites | Features | Notes |
| -------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | |
@ -13,6 +13,7 @@ Scrape a booru-like post object from the DOM.
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | |
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net, www.pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |

1
src/NeoScraper.ts
View File

@ -16,6 +16,7 @@ export default class NeoScraper {
new engines.Gelbooru(),
new engines.Inkbunny(),
new engines.Moebooru(),
new engines.Philomena(),
new engines.Pixiv(),
new engines.Reddit(),
new engines.Rule34us(),

30
src/engines/Philomena.test.ts Normal file
View File

@ -0,0 +1,30 @@
import { scrapeUrl } from "../test/engine.js";
test("get derpibooru.org post 3153524", async () => {
const pageUrl = "https://derpibooru.org/images/3153524";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0]).not.toHaveTag("safe");
expect(res.posts[0]).toHaveTag("yay");
expect(res.posts[0]).toHaveTag("flutteryay");
expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640");
expect(res.posts[0]).toHaveResolution([4000, 4000]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg");
});
test("get derpibooru.org post 3157356", async () => {
const pageUrl = "https://derpibooru.org/images/3157356?q=oc";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0]).not.toHaveTag("safe");
expect(res.posts[0]).toHaveTag("oc");
expect(res.posts[0]).toHaveTag("serra20", "artist");
expect(res.posts[0]).toHaveTag("pony", "species");
expect(res.posts[0]).toHaveTag("eris_(oc)", "character");
expect(res.posts[0]).toHaveResolution([960, 960]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/5/3157356.png");
});

125
src/engines/Philomena.ts Normal file
View File

@ -0,0 +1,125 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { SafetyRating, TagCategory } from "../BooruTypes.js";
import { parseResolutionString } from "../Utility.js";
export default class Philomena extends ScrapeEngineBase {
name = "Philomena";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
notes = [];
supportedHosts = [
"derpibooru.org",
"trixiebooru.org",
"ponybooru.org",
"furbooru.org",
"ponerpics.org",
"manebooru.art",
"twibooru.org",
];
removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"];
removePrefixesSuffix = new Map([["oc", "_(oc)"]]);
artistPrefixes = ["artist", "editor"];
safetyMap = new Map<string, SafetyRating>([
["safe", "safe"],
["suggestive", "sketchy"],
["questionable", "sketchy"],
["explicit", "unsafe"],
]);
categoryMap = new Map<string, TagCategory>([
["character", "character"],
["species", "species"],
["oc", "character"],
["content-official", "copyright"],
]);
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
const downloadEl = document.querySelector("a[title='Download (no tags in filename)']") as HTMLAnchorElement;
post.contentUrl = downloadEl?.href;
// Set content type
if (document.querySelector("video#image-display") != null) {
post.contentType = "video";
}
// Set default rating (Will be determined later)
post.rating = "unsafe";
// Set resolution
const resStr = (<HTMLElement>document.querySelector("span[class='image-size']"))?.innerText?.split(" ")[0];
if (resStr) {
const res = parseResolutionString(resStr);
if (res) {
post.resolution = res;
}
}
// Set tags
const tagEls = Array.from(document.querySelectorAll(".tag.dropdown")).map((x) => x as HTMLSpanElement);
for (const el of tagEls) {
let tagName = el.getAttribute("data-tag-name");
if (!tagName) continue;
// Set rating (philomena uses tags for this)
// These tags also have `data-tag-category="rating"` which we could use.
const safety = this.safetyMap.get(tagName);
if (safety != undefined) {
post.rating = safety;
// Don't add this as a tag.
continue;
}
let category: TagCategory | undefined;
// Categorize tags using data-tag-category
const dataTagCategory = el.getAttribute("data-tag-category");
if (dataTagCategory) {
const mappedCategory = this.categoryMap.get(dataTagCategory);
if (mappedCategory) {
category = mappedCategory;
}
}
// Artist tag categorize using prefixes
for (const prefix of this.artistPrefixes) {
if (tagName.includes(prefix + ":")) {
category = "artist";
break;
}
}
// Remove listed prefixes (removePrefixes) from tagName
for (const prefix of this.removePrefixes) {
if (tagName.includes(prefix + ":")) {
tagName = tagName.replace(prefix + ":", "").trim();
// Add suffix if configured
const suffix = this.removePrefixesSuffix.get(prefix);
if (suffix) {
tagName = tagName + suffix;
}
break;
}
}
// Push tag
if (tagName) {
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
// Set source
const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement);
post.sources = sourceEls.map((x) => x.href);
result.tryAddPost(post);
return result;
}
}

1
src/engines/index.ts
View File

@ -7,6 +7,7 @@ export { default as FurAffinity } from "./FurAffinity.js";
export { default as Gelbooru } from "./Gelbooru.js";
export { default as Inkbunny } from "./Inkbunny.js";
export { default as Moebooru } from "./Moebooru.js";
export { default as Philomena } from "./Philomena.js";
export { default as Pixiv } from "./Pixiv.js";
export { default as Reddit } from "./Reddit.js";
export { default as Rule34us } from "./Rule34us.js";