Update neo-scraper and add experimental support for e-hentai
parent
96466b9199
commit
c0ac677794
1
.npmrc
1
.npmrc
|
|
@ -1,2 +1,3 @@
|
|||
shamefully-hoist=true
|
||||
auto-install-peers=true
|
||||
strict-peer-dependencies=false
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@
|
|||
"deepmerge": "^4.3.1",
|
||||
"lodash": "^4.17.21",
|
||||
"markdown-it": "^13.0.1",
|
||||
"neo-scraper": "^0.5.8",
|
||||
"neo-scraper": "^0.6.0",
|
||||
"normalize.css": "^8.0.1",
|
||||
"pinia": "^2.0.36",
|
||||
"vue-router": "4"
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ specifiers:
|
|||
kolorist: ^1.6.0
|
||||
lodash: ^4.17.21
|
||||
markdown-it: ^13.0.1
|
||||
neo-scraper: ^0.5.8
|
||||
neo-scraper: ^0.6.0
|
||||
normalize.css: ^8.0.1
|
||||
npm-run-all: ^4.1.5
|
||||
pinia: ^2.0.36
|
||||
|
|
@ -55,7 +55,7 @@ dependencies:
|
|||
deepmerge: 4.3.1
|
||||
lodash: 4.17.21
|
||||
markdown-it: 13.0.1
|
||||
neo-scraper: 0.5.8
|
||||
neo-scraper: 0.6.0
|
||||
normalize.css: 8.0.1
|
||||
pinia: 2.0.36_hmuptsblhheur2tugfgucj7gc4
|
||||
vue-router: 4.1.6_vue@3.2.47
|
||||
|
|
@ -3960,6 +3960,11 @@ packages:
|
|||
nopt: 6.0.0
|
||||
dev: true
|
||||
|
||||
/js-cookie/3.0.5:
|
||||
resolution: {integrity: sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==}
|
||||
engines: {node: '>=14'}
|
||||
dev: false
|
||||
|
||||
/js-sdsl/4.4.0:
|
||||
resolution: {integrity: sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==}
|
||||
dev: true
|
||||
|
|
@ -4590,9 +4595,10 @@ packages:
|
|||
dev: true
|
||||
optional: true
|
||||
|
||||
/neo-scraper/0.5.8:
|
||||
resolution: {integrity: sha512-81SUT7Fo1o7z6HzbhT+GdGY1pAo+1E+s2WmFm08WDe02n9crpRy4cOFYdTtINa/W224YtMvKVC23YD6eLAlayQ==}
|
||||
/neo-scraper/0.6.0:
|
||||
resolution: {integrity: sha512-ytvdnj4NEaCw9KS+Zgg1lv5t/8l5Sq0jsDVW4OykvWQs4W35skcEnYQyL/LnKCmQ7aU5ZywIeVllvh8LiAZgcg==}
|
||||
dependencies:
|
||||
js-cookie: 3.0.5
|
||||
turndown: 7.1.2
|
||||
dev: false
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
SetPostUploadInfoData,
|
||||
SetExactPostId,
|
||||
PostUpdateCommandData,
|
||||
FetchCommandData,
|
||||
} from "~/models";
|
||||
import { PostAlreadyUploadedError } from "~/api/models";
|
||||
import SzurubooruApi from "~/api";
|
||||
|
|
@ -129,6 +130,15 @@ async function updatePost(data: PostUpdateCommandData) {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes fetch in the background page. This allows us to do "forbidden" stuff, like ignoring CORS headers.
|
||||
* @param data
|
||||
* @returns
|
||||
*/
|
||||
async function executeFetch(data: FetchCommandData) {
|
||||
return await fetch(data.url, data.options);
|
||||
}
|
||||
|
||||
async function messageHandler(cmd: BrowserCommand): Promise<any> {
|
||||
console.log("Background received message:");
|
||||
console.dir(cmd);
|
||||
|
|
@ -138,6 +148,8 @@ async function messageHandler(cmd: BrowserCommand): Promise<any> {
|
|||
return uploadPost(cmd.data);
|
||||
case "update_post":
|
||||
return updatePost(cmd.data);
|
||||
case "fetch":
|
||||
return executeFetch(cmd.data);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ export class ScrapedPostDetails {
|
|||
tags: TagDetails[] = [];
|
||||
notes: ScrapedNote[];
|
||||
contentUrl: string;
|
||||
extraContentUrl: string | undefined;
|
||||
contentSize: number | undefined;
|
||||
pageUrl: string;
|
||||
contentType: ContentType;
|
||||
|
|
@ -59,6 +60,7 @@ export class ScrapedPostDetails {
|
|||
|
||||
constructor(post: ScrapedPost) {
|
||||
this.contentUrl = post.contentUrl;
|
||||
this.extraContentUrl = post.extraContentUrl;
|
||||
// this.contentSize = post.contentSize;
|
||||
this.pageUrl = post.pageUrl;
|
||||
this.contentType = post.contentType;
|
||||
|
|
@ -104,7 +106,8 @@ export type BrowserCommandName =
|
|||
| "set_post_upload_info"
|
||||
| "set_exact_post_id"
|
||||
| "update_post"
|
||||
| "set_post_update_info";
|
||||
| "set_post_update_info"
|
||||
| "fetch";
|
||||
|
||||
export class BrowserCommand<T = any> {
|
||||
name: BrowserCommandName;
|
||||
|
|
@ -139,3 +142,7 @@ export class PostUpdateCommandData {
|
|||
public readonly selectedSite: SzuruSiteConfig
|
||||
) {}
|
||||
}
|
||||
|
||||
export class FetchCommandData {
|
||||
constructor(public readonly url: string, public readonly options: RequestInit | undefined = undefined) {}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import SzurubooruApi from "~/api";
|
|||
|
||||
const pop = usePopupStore();
|
||||
const isSearchingForSimilarPosts = ref<number>(0);
|
||||
const enableAutoSearch = ref(true);
|
||||
|
||||
const selectedSite = computed(() => {
|
||||
if (cfg.value.selectedSiteId) {
|
||||
|
|
@ -46,7 +47,7 @@ watch(
|
|||
() => pop.selectedPostId,
|
||||
(value) => {
|
||||
// Call findSimilar if wanted.
|
||||
if (cfg.value.autoSearchSimilar) {
|
||||
if (cfg.value.autoSearchSimilar && enableAutoSearch.value) {
|
||||
let selectedPost = pop.posts.find((x) => x.id == value);
|
||||
if (selectedPost) findSimilar(selectedPost);
|
||||
}
|
||||
|
|
@ -120,13 +121,24 @@ async function grabPost() {
|
|||
}
|
||||
|
||||
if (pop.posts.length > 0) {
|
||||
// Hack so that findSimilar does not trigger when we set selectedPostId, due to the watch(...).
|
||||
// This because we might change the contentUrl in fetchPostInfo, after which we'd need to call findSimilar again.
|
||||
// In which case it would be better to just call it once.
|
||||
enableAutoSearch.value = false;
|
||||
pop.selectedPostId = pop.posts[0].id;
|
||||
|
||||
if (cfg.value.loadTagCounts) {
|
||||
loadTagCounts();
|
||||
}
|
||||
|
||||
if (cfg.value.fetchPostInfo) {
|
||||
fetchPostInfo();
|
||||
await fetchPostInfo();
|
||||
}
|
||||
|
||||
// Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl.
|
||||
enableAutoSearch.value = true;
|
||||
if (cfg.value.autoSearchSimilar) {
|
||||
findSimilar(pop.selectedPost);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -272,11 +284,14 @@ async function loadTagCounts() {
|
|||
|
||||
async function fetchPostInfo() {
|
||||
for (const post of pop.posts) {
|
||||
if (!post.contentSize) {
|
||||
if (!post.contentSize || post.extraContentUrl) {
|
||||
// We are missing cookies/etc which means that the request might fail.
|
||||
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
|
||||
try {
|
||||
const res = await fetch(post.contentUrl, { method: "HEAD" });
|
||||
const contentUrl = post.extraContentUrl ?? post.contentUrl;
|
||||
|
||||
// TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup.
|
||||
const res = await fetch(contentUrl, { method: "HEAD" });
|
||||
const size = res.headers.get("Content-Length");
|
||||
const type = res.headers.get("Content-Type");
|
||||
|
||||
|
|
@ -286,8 +301,14 @@ async function fetchPostInfo() {
|
|||
const [_main, sub] = type.split("/");
|
||||
if (sub) post.contentSubType = sub.toUpperCase();
|
||||
}
|
||||
} catch {
|
||||
// TODO: Log errors.
|
||||
|
||||
// Resolve extraContentUrl redirects.
|
||||
if (post.extraContentUrl && res.url != post.contentUrl) {
|
||||
post.contentUrl = res.url;
|
||||
}
|
||||
} catch (ex) {
|
||||
// TODO: Maybe display an error in the UI.
|
||||
console.error(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue