diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index 3c98051..b1cb84f 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -10,7 +10,7 @@ export class ScrapedPost { rating: SafetyRating = "safe"; tags: ScrapedTag[] = []; notes: ScrapedNote[] = []; - source: string | undefined; + sources: string[] = []; referrer: string | undefined; } diff --git a/src/Utility.ts b/src/Utility.ts index 64f389f..2eb1c7b 100644 --- a/src/Utility.ts +++ b/src/Utility.ts @@ -120,6 +120,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement return undefined; } + // eslint-disable-next-line @typescript-eslint/no-explicit-any const data = (el).dataset as DOMStringMap; if (!data) { @@ -194,8 +195,8 @@ export function parseResolutionString(str: string | undefined | null): [number, // Example string: 1600x2200 const res = str .split("x") // Split on 'x' character - .map((v, _) => v.replace(/\D/g, "")) // Remove all non-digits - .map((v, _) => parseInt(v)) // Parse ints + .map((v) => v.replace(/\D/g, "")) // Remove all non-digits + .map((v) => parseInt(v)) // Parse ints .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0. if (res.length == 2) { diff --git a/src/engines/Danbooru.test.ts b/src/engines/Danbooru.test.ts index e559116..0dd3694 100644 --- a/src/engines/Danbooru.test.ts +++ b/src/engines/Danbooru.test.ts @@ -13,6 +13,7 @@ test("get danbooru.donmai.us post 3555872", async () => { expect(res.posts[0]).toHaveTag("t-shirt"); expect(res.posts[0]).toHaveTag("long_hair"); expect(res.posts[0]).toHaveResolution([1142, 1600]); + expect(res.posts[0]).toHaveSource("https://twitter.com/omatiosake/status/1124306601584672768"); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg" diff --git a/src/engines/Danbooru.ts b/src/engines/Danbooru.ts index 53767d0..a95ea5f 100644 --- a/src/engines/Danbooru.ts +++ b/src/engines/Danbooru.ts @@ -101,7 +101,7 @@ export default class Danbooru extends ScrapeEngineBase { if (sourceEl && sourceEl.children.length > 0) { const sourceLink = sourceEl.children[0] as HTMLAnchorElement; if (sourceLink) { - post.source = sourceLink.href; + post.sources = [sourceLink.href]; } } diff --git a/src/engines/Gelbooru.test.ts b/src/engines/Gelbooru.test.ts index b48cbb0..b3758b0 100644 --- a/src/engines/Gelbooru.test.ts +++ b/src/engines/Gelbooru.test.ts @@ -25,6 +25,7 @@ test("get gelbooru.com post 4925055", async () => { expect(res.posts[0]).toHaveTag("maosen", "artist"); expect(res.posts[0]).toHaveTag("butterfly"); expect(res.posts[0]).toHaveResolution([720, 1103]); + expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958"); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); }); @@ -45,6 +46,7 @@ test("get gelbooru.com post 6225495 with notes", async () => { expect(res.posts[0]).toHaveNote("But no need to worry!"); expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!"); expect(res.posts[0]).toHaveResolution([1600, 2200]); + expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816"); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png"); }); diff --git a/src/engines/Gelbooru.ts b/src/engines/Gelbooru.ts index 703ef27..0dc6040 100644 --- a/src/engines/Gelbooru.ts +++ b/src/engines/Gelbooru.ts @@ -80,9 +80,9 @@ export default class Gelbooru extends ScrapeEngineBase { // If Source element if (matches[1] == "Source") { if (version == Version.v020) { - post.source = matches[2]; + post.sources = [matches[2]]; } else { - post.source = (el.children[0] as HTMLAnchorElement).href; + post.sources = [(el.children[0] as HTMLAnchorElement).href]; } } // If Rating element diff --git a/src/engines/Moebooru.test.ts b/src/engines/Moebooru.test.ts index d2f7b2c..7f5e5e2 100644 --- a/src/engines/Moebooru.test.ts +++ b/src/engines/Moebooru.test.ts @@ -21,6 +21,7 @@ test("get yande.re post 684478 with notes", async () => { expect(res.posts[0]).toHaveTag("gym_uniform"); expect(res.posts[0]).toHaveNote("post #671879"); expect(res.posts[0]).toHaveResolution([2480, 3508]); + expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/84228323"); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg" @@ -33,6 +34,7 @@ test("get konachan.com post 289968", async () => { expect(res.posts.length).toBe(1); expect(res.posts[0]).toHaveTag("league_of_legends", "copyright"); expect(res.posts[0]).toHaveResolution([1200, 783]); + expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72418059"); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg" @@ -49,6 +51,7 @@ test("get konachan.com post 327454 with notes", async () => { expect(res.posts[0]).toHaveTag("water"); expect(res.posts[0]).toHaveNote("Under construction. \nBe careful of your safety"); expect(res.posts[0]).toHaveResolution([5629, 2505]); + expect(res.posts[0]).toHaveSource("https://www.pixiv.net/en/artworks/86644963"); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg" @@ -71,4 +74,5 @@ test("get konachan.com post 285782 with notes", async () => { expect(res.posts[0].contentUrl).toBe( "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg" ); + expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=75747354"); }); diff --git a/src/engines/Moebooru.ts b/src/engines/Moebooru.ts index 606c66e..76795de 100644 --- a/src/engines/Moebooru.ts +++ b/src/engines/Moebooru.ts @@ -35,7 +35,7 @@ export default class Moebooru extends ScrapeEngineBase { if (matches && matches.length == 3) { // If Source element if (matches[1] == "Source") { - post.source = (el.children[0] as HTMLAnchorElement).href; + post.sources = [(el.children[0] as HTMLAnchorElement).href]; } // If Rating element else if (matches[1] == "Rating") { diff --git a/src/engines/Shimmie2.test.ts b/src/engines/Shimmie2.test.ts index c07d304..e3e1a9a 100644 --- a/src/engines/Shimmie2.test.ts +++ b/src/engines/Shimmie2.test.ts @@ -5,6 +5,9 @@ test("get rule34.paheal.net post 2505982", async () => { const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0]).toHaveSource( + "http://law-zilla.tumblr.com/post/171249285285/finished-subdraw-37-kaisa-from-league-of-legends" + ); expect(res.posts[0].contentUrl).toContain( ".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png" ); diff --git a/src/engines/Shimmie2.ts b/src/engines/Shimmie2.ts index 592a4ae..27a7e02 100644 --- a/src/engines/Shimmie2.ts +++ b/src/engines/Shimmie2.ts @@ -44,7 +44,7 @@ export default class Shimmie2 extends ScrapeEngineBase { // Set source const sourceEl = document.querySelector(".image_info > tbody > tr > td > div > a") as HTMLAnchorElement; if (sourceEl) { - post.source = sourceEl.href; + post.sources = [sourceEl.href]; } result.tryAddPost(post); diff --git a/src/engines/e621.test.ts b/src/engines/e621.test.ts index f87ffa1..79bd28d 100644 --- a/src/engines/e621.test.ts +++ b/src/engines/e621.test.ts @@ -9,6 +9,7 @@ test("get e621.net post 380315", async () => { expect(res.posts[0]).toHaveTag("ambiguous_gender"); expect(res.posts[0]).toHaveTag("low_res", "meta"); expect(res.posts[0]).toHaveResolution([500, 500]); + expect(res.posts[0]).toHaveSource("https://inkinesss.deviantart.com/art/Nasus-388347996"); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe("https://static1.e621.net/data/92/e7/92e71bd73ab219d80516c4d9adad3db5.jpg"); }); diff --git a/src/engines/e621.ts b/src/engines/e621.ts index 0d181db..caf1b8a 100644 --- a/src/engines/e621.ts +++ b/src/engines/e621.ts @@ -82,10 +82,7 @@ export default class e621 extends ScrapeEngineBase { // Set source const sourceEls = Array.from(document.querySelectorAll(".source-link > a")).map((x) => x as HTMLAnchorElement); - // We currently only support one source. - if (sourceEls.length > 0) { - post.source = sourceEls[0].href; - } + post.sources = sourceEls.map((x) => x.href); result.tryAddPost(post); diff --git a/src/test/jest.d.ts b/src/test/jest.d.ts index 3fbfb73..af74943 100644 --- a/src/test/jest.d.ts +++ b/src/test/jest.d.ts @@ -5,9 +5,10 @@ declare global { namespace jest { interface Matchers { // The `= undefined` part is incorrect according to VSCode, but it is actually required. - toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): R; - toHaveNote(noteText: string): R; - toHaveResolution(resolution: [number, number]): R; + toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult; + toHaveNote(noteText: string): CustomMatcherResult; + toHaveResolution(resolution: [number, number]): CustomMatcherResult; + toHaveSource(source: string): CustomMatcherResult; } } } diff --git a/src/test/jest.setup.ts b/src/test/jest.setup.ts index 8e43229..64750f8 100644 --- a/src/test/jest.setup.ts +++ b/src/test/jest.setup.ts @@ -52,4 +52,13 @@ expect.extend({ } return { pass, message }; }, + toHaveSource(received: any, source: string): jest.CustomMatcherResult { + let pass = false; + const message = () => (pass ? "" : `Post does not have source '${source}'.`); + + if (received instanceof ScrapedPost) { + pass = received.sources.find((x) => x == source) != undefined; + } + return { pass, message }; + }, });