From c0ac677794341b06fca5a8362dbdb185bfa0d75f Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Mon, 19 Jun 2023 21:38:43 +0200 Subject: [PATCH] Update neo-scraper and add experimental support for e-hentai --- .npmrc | 1 + package.json | 2 +- pnpm-lock.yaml | 14 ++++++++++---- src/background/main.ts | 16 ++++++++++++++-- src/models/index.ts | 9 ++++++++- src/popup/pages/PopupMain.vue | 33 +++++++++++++++++++++++++++------ 6 files changed, 61 insertions(+), 14 deletions(-) diff --git a/.npmrc b/.npmrc index 009aa06..c82236b 100644 --- a/.npmrc +++ b/.npmrc @@ -1,2 +1,3 @@ shamefully-hoist=true auto-install-peers=true +strict-peer-dependencies=false diff --git a/package.json b/package.json index 62f144f..ee107b1 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "deepmerge": "^4.3.1", "lodash": "^4.17.21", "markdown-it": "^13.0.1", - "neo-scraper": "^0.5.8", + "neo-scraper": "^0.6.0", "normalize.css": "^8.0.1", "pinia": "^2.0.36", "vue-router": "4" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 06fb67b..6ea523a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,7 +31,7 @@ specifiers: kolorist: ^1.6.0 lodash: ^4.17.21 markdown-it: ^13.0.1 - neo-scraper: ^0.5.8 + neo-scraper: ^0.6.0 normalize.css: ^8.0.1 npm-run-all: ^4.1.5 pinia: ^2.0.36 @@ -55,7 +55,7 @@ dependencies: deepmerge: 4.3.1 lodash: 4.17.21 markdown-it: 13.0.1 - neo-scraper: 0.5.8 + neo-scraper: 0.6.0 normalize.css: 8.0.1 pinia: 2.0.36_hmuptsblhheur2tugfgucj7gc4 vue-router: 4.1.6_vue@3.2.47 @@ -3960,6 +3960,11 @@ packages: nopt: 6.0.0 dev: true + /js-cookie/3.0.5: + resolution: {integrity: sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==} + engines: {node: '>=14'} + dev: false + /js-sdsl/4.4.0: resolution: {integrity: sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==} dev: true @@ -4590,9 +4595,10 @@ packages: dev: true optional: true - /neo-scraper/0.5.8: - resolution: {integrity: sha512-81SUT7Fo1o7z6HzbhT+GdGY1pAo+1E+s2WmFm08WDe02n9crpRy4cOFYdTtINa/W224YtMvKVC23YD6eLAlayQ==} + /neo-scraper/0.6.0: + resolution: {integrity: sha512-ytvdnj4NEaCw9KS+Zgg1lv5t/8l5Sq0jsDVW4OykvWQs4W35skcEnYQyL/LnKCmQ7aU5ZywIeVllvh8LiAZgcg==} dependencies: + js-cookie: 3.0.5 turndown: 7.1.2 dev: false diff --git a/src/background/main.ts b/src/background/main.ts index 1812ad3..7bbee75 100644 --- a/src/background/main.ts +++ b/src/background/main.ts @@ -6,6 +6,7 @@ import { SetPostUploadInfoData, SetExactPostId, PostUpdateCommandData, + FetchCommandData, } from "~/models"; import { PostAlreadyUploadedError } from "~/api/models"; import SzurubooruApi from "~/api"; @@ -115,11 +116,11 @@ async function updatePost(data: PostUpdateCommandData) { try { const szuru = SzurubooruApi.createFromConfig(data.selectedSite); - + pushInfo(); await szuru.updatePost(data.postId, data.updateRequest); - + info.state = "uploaded"; pushInfo(); } catch (ex: any) { @@ -129,6 +130,15 @@ async function updatePost(data: PostUpdateCommandData) { } } +/** + * Executes fetch in the background page. This allows us to do "forbidden" stuff, like ignoring CORS headers. + * @param data + * @returns + */ +async function executeFetch(data: FetchCommandData) { + return await fetch(data.url, data.options); +} + async function messageHandler(cmd: BrowserCommand): Promise { console.log("Background received message:"); console.dir(cmd); @@ -138,6 +148,8 @@ async function messageHandler(cmd: BrowserCommand): Promise { return uploadPost(cmd.data); case "update_post": return updatePost(cmd.data); + case "fetch": + return executeFetch(cmd.data); } } diff --git a/src/models/index.ts b/src/models/index.ts index b1a68af..dc2672d 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -47,6 +47,7 @@ export class ScrapedPostDetails { tags: TagDetails[] = []; notes: ScrapedNote[]; contentUrl: string; + extraContentUrl: string | undefined; contentSize: number | undefined; pageUrl: string; contentType: ContentType; @@ -59,6 +60,7 @@ export class ScrapedPostDetails { constructor(post: ScrapedPost) { this.contentUrl = post.contentUrl; + this.extraContentUrl = post.extraContentUrl; // this.contentSize = post.contentSize; this.pageUrl = post.pageUrl; this.contentType = post.contentType; @@ -104,7 +106,8 @@ export type BrowserCommandName = | "set_post_upload_info" | "set_exact_post_id" | "update_post" - | "set_post_update_info"; + | "set_post_update_info" + | "fetch"; export class BrowserCommand { name: BrowserCommandName; @@ -139,3 +142,7 @@ export class PostUpdateCommandData { public readonly selectedSite: SzuruSiteConfig ) {} } + +export class FetchCommandData { + constructor(public readonly url: string, public readonly options: RequestInit | undefined = undefined) {} +} diff --git a/src/popup/pages/PopupMain.vue b/src/popup/pages/PopupMain.vue index 53d0b06..4916de8 100644 --- a/src/popup/pages/PopupMain.vue +++ b/src/popup/pages/PopupMain.vue @@ -21,6 +21,7 @@ import SzurubooruApi from "~/api"; const pop = usePopupStore(); const isSearchingForSimilarPosts = ref(0); +const enableAutoSearch = ref(true); const selectedSite = computed(() => { if (cfg.value.selectedSiteId) { @@ -46,7 +47,7 @@ watch( () => pop.selectedPostId, (value) => { // Call findSimilar if wanted. - if (cfg.value.autoSearchSimilar) { + if (cfg.value.autoSearchSimilar && enableAutoSearch.value) { let selectedPost = pop.posts.find((x) => x.id == value); if (selectedPost) findSimilar(selectedPost); } @@ -120,13 +121,24 @@ async function grabPost() { } if (pop.posts.length > 0) { + // Hack so that findSimilar does not trigger when we set selectedPostId, due to the watch(...). + // This because we might change the contentUrl in fetchPostInfo, after which we'd need to call findSimilar again. + // In which case it would be better to just call it once. + enableAutoSearch.value = false; pop.selectedPostId = pop.posts[0].id; + if (cfg.value.loadTagCounts) { loadTagCounts(); } if (cfg.value.fetchPostInfo) { - fetchPostInfo(); + await fetchPostInfo(); + } + + // Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl. + enableAutoSearch.value = true; + if (cfg.value.autoSearchSimilar) { + findSimilar(pop.selectedPost); } } } @@ -272,11 +284,14 @@ async function loadTagCounts() { async function fetchPostInfo() { for (const post of pop.posts) { - if (!post.contentSize) { + if (!post.contentSize || post.extraContentUrl) { // We are missing cookies/etc which means that the request might fail. // If you want access to the cookies/session/etc then you need to execute this code inside the content script. try { - const res = await fetch(post.contentUrl, { method: "HEAD" }); + const contentUrl = post.extraContentUrl ?? post.contentUrl; + + // TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup. + const res = await fetch(contentUrl, { method: "HEAD" }); const size = res.headers.get("Content-Length"); const type = res.headers.get("Content-Type"); @@ -286,8 +301,14 @@ async function fetchPostInfo() { const [_main, sub] = type.split("/"); if (sub) post.contentSubType = sub.toUpperCase(); } - } catch { - // TODO: Log errors. + + // Resolve extraContentUrl redirects. + if (post.extraContentUrl && res.url != post.contentUrl) { + post.contentUrl = res.url; + } + } catch (ex) { + // TODO: Maybe display an error in the UI. + console.error(ex); } } }