diff --git a/package.json b/package.json index 26ae99a..cee197c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "neo-scraper", - "version": "0.4.7", + "version": "0.5.0", "description": "Scrape a booru-like post object from the DOM.", "main": "dist/index.js", "types": "dist/index.d.js", @@ -15,6 +15,7 @@ "devDependencies": { "@types/jest": "^24.9.1", "@types/jsdom": "^12.2.4", + "@types/turndown": "^5.0.1", "jest": "^24.9.0", "jsdom": "^15.2.1", "ts-jest": "^24.3.0", @@ -29,5 +30,8 @@ "js" ], "testRegex": "^.+\\.test\\.ts$" + }, + "dependencies": { + "turndown": "^7.1.1" } } diff --git a/src/NeoScraper.test.ts b/src/NeoScraper.test.ts index 5e8c420..7021669 100644 --- a/src/NeoScraper.test.ts +++ b/src/NeoScraper.test.ts @@ -46,10 +46,22 @@ test("get danbooru.donmai.us post 3555872", async () => { expect(res.posts.length).toBe(1); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( - "https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg" + "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg" ); }); +test("get danbooru.donmai.us post 4924490", async () => { + var pageUrl = "https://danbooru.donmai.us/posts/4924490"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png" + ); + expect(res.posts[0].notes).toHaveLength(17); + expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪"); +}); + // Gelbooru test("get safebooru.org post 2871532", async () => { @@ -67,7 +79,7 @@ test("get gelbooru.com post 4925055", async () => { const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].contentUrl).toBe("https://img2.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); + expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); }); test("get rule34.xxx post 2740994", async () => { @@ -110,13 +122,13 @@ test("get old.reddit.com post", async () => { expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); }); -test("get new.reddit.com post", async () => { - var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); -}); +// test("get new.reddit.com post", async () => { +// var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; +// const res = await scrapeUrl(pageUrl); +// expect(res.posts.length).toBe(1); +// expect(res.posts[0].pageUrl).toBe(pageUrl); +// expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); +// }); // Shimmie2 diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index c6ed1d1..0427d41 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -6,14 +6,16 @@ export class ScrapedPost { contentUrl: string = ""; pageUrl: string = ""; contentType: ContentType = "image"; + resolution: [number, number] | undefined; rating: SafetyRating = "safe"; tags: ScrapedTag[] = []; - source: string | undefined = undefined; - referrer: string | undefined = undefined; + notes: ScrapedNote[] = []; + source: string | undefined; + referrer: string | undefined; } export class ScrapedTag { - name: string = ""; + name: string; category?: TagCategory; constructor(name: string, category?: TagCategory) { @@ -22,6 +24,10 @@ export class ScrapedTag { } } +export class ScrapedNote { + constructor(public text: string, public polygons: number[][]) {} +} + export class ScrapeResult { engine: string; description: string = ""; @@ -59,3 +65,13 @@ export interface ScrapeEngine { canImport(url: Location): boolean; scrapeDocument(document: Document): ScrapeResult; } + +export abstract class ScrapeEngineBase implements ScrapeEngine { + abstract name: string; + abstract canImport(url: Location): boolean; + abstract scrapeDocument(document: Document): ScrapeResult; + + protected log(str: string) { + console.log(`[${this.name}] ${str}`); + } +} diff --git a/src/Utility.ts b/src/Utility.ts index cf7595d..ec32d32 100644 --- a/src/Utility.ts +++ b/src/Utility.ts @@ -1,4 +1,5 @@ -import { ContentType } from "./ScrapeEngine"; +import TurndownService from "turndown"; +import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine"; // https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json export const videoExtensions = [ @@ -48,3 +49,88 @@ export function guessContentType(url: string): ContentType { } return "image"; } + +export function htmlNoteToMarkdownNote(text: string) { + return new TurndownService().turndown(text); +} + +export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement): ScrapedNote | undefined { + function logFail(str: string) { + console.log("[createNoteFromDanbooruArticle] Can't create note. " + str); + } + + function getInt(map: DOMStringMap, key: string) { + const value = map[key]; + if (!value) { + logFail(`Key '${key}' not found in data.`); + return undefined; + } + const num = parseInt(value); + if (num == NaN) { + logFail(`parseInt on '${value}' returned NaN.`); + } + return num; + } + + function validateNote(note: ScrapedNote) { + for (const poly of note.polygons) { + if (poly.some((x) => x > 1 || x < 0)) { + logFail("Polygon out of range."); + return false; + } + } + + return true; + } + + if (!post.resolution) { + logFail("Post resolution is undefined."); + return undefined; + } + + const data = (el).dataset as DOMStringMap; + + if (!data) { + logFail("Data is undefined."); + return undefined; + } + + let text = data["body"]; + if (text) { + text = htmlNoteToMarkdownNote(text); + } + + let x = getInt(data, "x"); + let y = getInt(data, "y"); + let w = getInt(data, "width"); + let h = getInt(data, "height"); + + if (!text || !x || !y || !w || !h) { + logFail("A required data field could not be parsed."); + return undefined; + } + + // Normalize our numbers from 0 to 1 + x = x / post.resolution[0]; + y = y / post.resolution[1]; + w = w / post.resolution[0]; + h = h / post.resolution[1]; + + const polygons = [ + // Top left + [x, y], + // Top right + [x + w, y], + // Bottom right + [x + w, y + h], + // Bottom left + [x, y + h], + ]; + + const note = new ScrapedNote(text, polygons); + if (validateNote(note)) { + return note; + } else { + return undefined; + } +} diff --git a/src/engines/Danbooru.ts b/src/engines/Danbooru.ts index 5dca812..b3fce29 100644 --- a/src/engines/Danbooru.ts +++ b/src/engines/Danbooru.ts @@ -1,8 +1,8 @@ -import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; -import { guessContentType } from "../Utility"; +import { createNoteFromDanbooruArticle, guessContentType } from "../Utility"; -export default class Danbooru implements ScrapeEngine { +export default class Danbooru extends ScrapeEngineBase { name = "danbooru"; canImport(url: Location): boolean { @@ -33,6 +33,16 @@ export default class Danbooru implements ScrapeEngine { post.contentType = guessContentType(post.contentUrl); } + // Set post resolution + // parseInt never throws an error, so we don't have to validate its input. + const w = parseInt(document.body.dataset["postImageWidth"]!); + const h = parseInt(document.body.dataset["postImageHeight"]!); + if (!w || !h) { + this.log("Couldn't set post resolution."); + } else { + post.resolution = [w, h]; + } + // Set rating const safetyExp = new RegExp("Rating: (.*)"); const safetyElements = Array.from(document.querySelectorAll("#post-information > ul > li")) @@ -96,6 +106,16 @@ export default class Danbooru implements ScrapeEngine { } } + // Try to load notes + const noteEls = Array.from(document.querySelectorAll("section#notes > article")).map((x) => x as HTMLElement); + + for (const el of noteEls) { + const note = createNoteFromDanbooruArticle(post, el); + if (note) { + post.notes.push(note); + } + } + result.tryAddPost(post); return result; diff --git a/src/engines/Gelbooru.ts b/src/engines/Gelbooru.ts index dc41d95..4d599b0 100644 --- a/src/engines/Gelbooru.ts +++ b/src/engines/Gelbooru.ts @@ -1,7 +1,7 @@ -import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; import { CategoryMap } from "./Common"; -import { guessContentType } from "../Utility"; +import { createNoteFromDanbooruArticle, guessContentType } from "../Utility"; // Small hack to avoid screwing with semver enum Version { @@ -9,7 +9,7 @@ enum Version { v025, // 0.2.5 } -export default class Gelbooru implements ScrapeEngine { +export default class Gelbooru extends ScrapeEngineBase { name = "gelbooru"; private readonly classNameToCategoryMap: CategoryMap = { @@ -42,7 +42,7 @@ export default class Gelbooru implements ScrapeEngine { return result; } - console.log("Gelbooru guessed version: " + version); + this.log("Guessed version: " + version); let post = new ScrapedPost(); post.pageUrl = document.location.href; @@ -105,6 +105,21 @@ export default class Gelbooru implements ScrapeEngine { break; } } + // If size element + else if (matches[1] == "Size") { + // Example string: 1600x2200 + const res = matches[2] + .split("x") // Split on 'x' character + .map((v, _) => parseInt(v)) // Parse ints + .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0. + + if (res.length == 2) { + // Kinda hacky, but this is currently the correct way to convert an array to a tuple. + post.resolution = [res[0], res[1]]; + } else { + this.log("Couldn't parse post resolution."); + } + } } } } @@ -154,6 +169,20 @@ export default class Gelbooru implements ScrapeEngine { } } + // Try to load notes + // NOTE: This code is exactly the same as in the Danbooru engine. + // Which means that when you fix a bug here, you might also want to fix it in the Danbooru engine. + // Unless the bug is only limited to Gelbooru based sites, of course. + const noteEls = Array.from(document.querySelectorAll("section#notes > article")).map((x) => x as HTMLElement); + + for (const el of noteEls) { + const note = createNoteFromDanbooruArticle(post, el); + if (note) { + post.notes.push(note); + } + } + // End of code duplication with Danbooru engine. + result.tryAddPost(post); return result; diff --git a/yarn.lock b/yarn.lock index 925d317..ca0cfb4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -453,6 +453,11 @@ resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.0.tgz#fef1904e4668b6e5ecee60c52cc6a078ffa6697d" integrity sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A== +"@types/turndown@^5.0.1": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@types/turndown/-/turndown-5.0.1.tgz#fcda7b02cda4c9d445be1440036df20f335b9387" + integrity sha512-N8Ad4e3oJxh9n9BiZx9cbe/0M3kqDpOTm2wzj13wdDUxDPjfjloWIJaquZzWE1cYTAHpjOH3rcTnXQdpEfS/SQ== + "@types/yargs-parser@*": version "20.2.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9" @@ -1029,6 +1034,11 @@ domexception@^1.0.1: dependencies: webidl-conversions "^4.0.2" +domino@^2.1.6: + version "2.1.6" + resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe" + integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ== + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -3250,9 +3260,9 @@ throat@^4.0.0: integrity sha1-iQN8vJLFarGJJua6TLsgDhVnKmo= tmpl@1.0.x: - version "1.0.4" - resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" - integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE= + version "1.0.5" + resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" + integrity "sha1-hoPguQK7nCDE9ybjwLafNlGMB8w= sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==" to-fast-properties@^2.0.0: version "2.0.0" @@ -3331,6 +3341,13 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +turndown@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.1.1.tgz#96992f2d9b40a1a03d3ea61ad31b5a5c751ef77f" + integrity sha512-BEkXaWH7Wh7e9bd2QumhfAXk5g34+6QUmmWx+0q6ThaVOLuLUqsnkq35HQ5SBHSaxjSfSM7US5o4lhJNH7B9MA== + dependencies: + domino "^2.1.6" + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"