From abda6473355a70c41fec04b1dc8cc91a63f9a569 Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Sat, 4 Jan 2020 17:30:01 +0100 Subject: [PATCH] Cleanup + fix issue where zerochan sometimes couldn't find the image --- src/NeoScraper.ts | 4 ++++ src/ScrapeEngine.ts | 8 ++++++++ src/engines/Danbooru.ts | 4 +--- src/engines/Gelbooru.ts | 4 +--- src/engines/Moebooru.ts | 5 +---- src/engines/Reddit.ts | 4 +--- src/engines/SankakuComplex.ts | 4 +--- src/engines/Shimmie2.ts | 4 +--- src/engines/Twitter.ts | 4 +--- src/engines/Zerochan.ts | 11 ++++++++--- 10 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index b5895db..3ac6bd5 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -26,6 +26,7 @@ export default class NeoScraper { for (const engine of this.engines) { if (ignoreCanImport || engine.canImport(document.location)) { + console.log("Using engine: " + engine.name); res.results.push(engine.scrapeDocument(document)); } } @@ -34,10 +35,13 @@ export default class NeoScraper { if (!this.fallbackEngine) { console.error("NeoScraper.fallbackEngine is unset!"); } else { + console.log("Using fallback engine"); res.results.push(this.fallbackEngine.scrapeDocument(document)); } } + console.dir(res); + return res; } } diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index fca73ef..4cb3175 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -25,6 +25,14 @@ export class ScrapeResult { constructor(engine: string) { this.engine = engine; } + + tryAddPost(post: ScrapedPost) { + if (!post.imageUrl) { + console.log(`[${this.engine}] Not adding post because imageUrl is unset!`); + } else { + this.posts.push(post); + } + } } export class ScrapeResults { diff --git a/src/engines/Danbooru.ts b/src/engines/Danbooru.ts index 8e24e4b..f9a9d79 100644 --- a/src/engines/Danbooru.ts +++ b/src/engines/Danbooru.ts @@ -78,9 +78,7 @@ export default class Danbooru implements ScrapeEngine { } } - if (post.imageUrl) { - result.posts.push(post); - } + result.tryAddPost(post); return result; } diff --git a/src/engines/Gelbooru.ts b/src/engines/Gelbooru.ts index 9adf1df..0d58c59 100644 --- a/src/engines/Gelbooru.ts +++ b/src/engines/Gelbooru.ts @@ -135,9 +135,7 @@ export default class Gelbooru implements ScrapeEngine { } } - if (post.imageUrl) { - result.posts.push(post); - } + result.tryAddPost(post); return result; } diff --git a/src/engines/Moebooru.ts b/src/engines/Moebooru.ts index 4870ad8..34e7422 100644 --- a/src/engines/Moebooru.ts +++ b/src/engines/Moebooru.ts @@ -88,10 +88,7 @@ export default class Moebooru implements ScrapeEngine { post.tags.push(tag); } - - if (post.imageUrl) { - result.posts.push(post); - } + result.tryAddPost(post); return result; } diff --git a/src/engines/Reddit.ts b/src/engines/Reddit.ts index 307a003..bbd18ac 100644 --- a/src/engines/Reddit.ts +++ b/src/engines/Reddit.ts @@ -25,9 +25,7 @@ export default class Reddit implements ScrapeEngine { post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href; } - if (post.imageUrl) { - result.posts.push(post); - } + result.tryAddPost(post); return result; } diff --git a/src/engines/SankakuComplex.ts b/src/engines/SankakuComplex.ts index 3bb0612..ad66f26 100644 --- a/src/engines/SankakuComplex.ts +++ b/src/engines/SankakuComplex.ts @@ -74,9 +74,7 @@ export default class SankakuComplex implements ScrapeEngine { } } - if (post.imageUrl) { - result.posts.push(post); - } + result.tryAddPost(post); return result; } diff --git a/src/engines/Shimmie2.ts b/src/engines/Shimmie2.ts index c34e271..77e212d 100644 --- a/src/engines/Shimmie2.ts +++ b/src/engines/Shimmie2.ts @@ -41,9 +41,7 @@ export default class Shimmie2 implements ScrapeEngine { .map(x => (x as HTMLAnchorElement).innerText.toLowerCase()) .map(x => new ScrapedTag(x)); - if (post.imageUrl) { - result.posts.push(post); - } + result.tryAddPost(post); return result; } diff --git a/src/engines/Twitter.ts b/src/engines/Twitter.ts index 3d3bc36..3b3af8a 100644 --- a/src/engines/Twitter.ts +++ b/src/engines/Twitter.ts @@ -33,9 +33,7 @@ export default class Twitter implements ScrapeEngine { post.imageUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src); } - if (post.imageUrl) { - result.posts.push(post); - } + result.tryAddPost(post); return result; } diff --git a/src/engines/Zerochan.ts b/src/engines/Zerochan.ts index ee677f4..4d263bb 100644 --- a/src/engines/Zerochan.ts +++ b/src/engines/Zerochan.ts @@ -16,7 +16,14 @@ export default class Zerochan implements ScrapeEngine { // Set image url const previewElements = document.getElementsByClassName("preview"); if (previewElements.length > 0) { + // The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version. post.imageUrl = (previewElements[0] as HTMLAnchorElement).href; + } else { + // When there is no larger version available just use the displayed version. + const imgElement = document.querySelector("img[title='No larger size available']") as HTMLImageElement; + if (imgElement) { + post.imageUrl = imgElement.src; + } } // Set rating @@ -59,9 +66,7 @@ export default class Zerochan implements ScrapeEngine { } } - if (post.imageUrl) { - result.posts.push(post); - } + result.tryAddPost(post); return result; }