Update neo-scraper and add experimental support for e-hentai

gh-actions
neobooru 2023-06-19 21:38:43 +02:00
parent 96466b9199
commit c0ac677794
6 changed files with 61 additions and 14 deletions

1
.npmrc
View File

@ -1,2 +1,3 @@
shamefully-hoist=true
auto-install-peers=true
strict-peer-dependencies=false

View File

@ -32,7 +32,7 @@
"deepmerge": "^4.3.1",
"lodash": "^4.17.21",
"markdown-it": "^13.0.1",
"neo-scraper": "^0.5.8",
"neo-scraper": "^0.6.0",
"normalize.css": "^8.0.1",
"pinia": "^2.0.36",
"vue-router": "4"

View File

@ -31,7 +31,7 @@ specifiers:
kolorist: ^1.6.0
lodash: ^4.17.21
markdown-it: ^13.0.1
neo-scraper: ^0.5.8
neo-scraper: ^0.6.0
normalize.css: ^8.0.1
npm-run-all: ^4.1.5
pinia: ^2.0.36
@ -55,7 +55,7 @@ dependencies:
deepmerge: 4.3.1
lodash: 4.17.21
markdown-it: 13.0.1
neo-scraper: 0.5.8
neo-scraper: 0.6.0
normalize.css: 8.0.1
pinia: 2.0.36_hmuptsblhheur2tugfgucj7gc4
vue-router: 4.1.6_vue@3.2.47
@ -3960,6 +3960,11 @@ packages:
nopt: 6.0.0
dev: true
/js-cookie/3.0.5:
resolution: {integrity: sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==}
engines: {node: '>=14'}
dev: false
/js-sdsl/4.4.0:
resolution: {integrity: sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==}
dev: true
@ -4590,9 +4595,10 @@ packages:
dev: true
optional: true
/neo-scraper/0.5.8:
resolution: {integrity: sha512-81SUT7Fo1o7z6HzbhT+GdGY1pAo+1E+s2WmFm08WDe02n9crpRy4cOFYdTtINa/W224YtMvKVC23YD6eLAlayQ==}
/neo-scraper/0.6.0:
resolution: {integrity: sha512-ytvdnj4NEaCw9KS+Zgg1lv5t/8l5Sq0jsDVW4OykvWQs4W35skcEnYQyL/LnKCmQ7aU5ZywIeVllvh8LiAZgcg==}
dependencies:
js-cookie: 3.0.5
turndown: 7.1.2
dev: false

View File

@ -6,6 +6,7 @@ import {
SetPostUploadInfoData,
SetExactPostId,
PostUpdateCommandData,
FetchCommandData,
} from "~/models";
import { PostAlreadyUploadedError } from "~/api/models";
import SzurubooruApi from "~/api";
@ -129,6 +130,15 @@ async function updatePost(data: PostUpdateCommandData) {
}
}
/**
* Executes fetch in the background page. This allows us to do "forbidden" stuff, like ignoring CORS headers.
* @param data
* @returns
*/
async function executeFetch(data: FetchCommandData) {
return await fetch(data.url, data.options);
}
async function messageHandler(cmd: BrowserCommand): Promise<any> {
console.log("Background received message:");
console.dir(cmd);
@ -138,6 +148,8 @@ async function messageHandler(cmd: BrowserCommand): Promise<any> {
return uploadPost(cmd.data);
case "update_post":
return updatePost(cmd.data);
case "fetch":
return executeFetch(cmd.data);
}
}

View File

@ -47,6 +47,7 @@ export class ScrapedPostDetails {
tags: TagDetails[] = [];
notes: ScrapedNote[];
contentUrl: string;
extraContentUrl: string | undefined;
contentSize: number | undefined;
pageUrl: string;
contentType: ContentType;
@ -59,6 +60,7 @@ export class ScrapedPostDetails {
constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl;
this.extraContentUrl = post.extraContentUrl;
// this.contentSize = post.contentSize;
this.pageUrl = post.pageUrl;
this.contentType = post.contentType;
@ -104,7 +106,8 @@ export type BrowserCommandName =
| "set_post_upload_info"
| "set_exact_post_id"
| "update_post"
| "set_post_update_info";
| "set_post_update_info"
| "fetch";
export class BrowserCommand<T = any> {
name: BrowserCommandName;
@ -139,3 +142,7 @@ export class PostUpdateCommandData {
public readonly selectedSite: SzuruSiteConfig
) {}
}
export class FetchCommandData {
constructor(public readonly url: string, public readonly options: RequestInit | undefined = undefined) {}
}

View File

@ -21,6 +21,7 @@ import SzurubooruApi from "~/api";
const pop = usePopupStore();
const isSearchingForSimilarPosts = ref<number>(0);
const enableAutoSearch = ref(true);
const selectedSite = computed(() => {
if (cfg.value.selectedSiteId) {
@ -46,7 +47,7 @@ watch(
() => pop.selectedPostId,
(value) => {
// Call findSimilar if wanted.
if (cfg.value.autoSearchSimilar) {
if (cfg.value.autoSearchSimilar && enableAutoSearch.value) {
let selectedPost = pop.posts.find((x) => x.id == value);
if (selectedPost) findSimilar(selectedPost);
}
@ -120,13 +121,24 @@ async function grabPost() {
}
if (pop.posts.length > 0) {
// Hack so that findSimilar does not trigger when we set selectedPostId, due to the watch(...).
// This because we might change the contentUrl in fetchPostInfo, after which we'd need to call findSimilar again.
// In which case it would be better to just call it once.
enableAutoSearch.value = false;
pop.selectedPostId = pop.posts[0].id;
if (cfg.value.loadTagCounts) {
loadTagCounts();
}
if (cfg.value.fetchPostInfo) {
fetchPostInfo();
await fetchPostInfo();
}
// Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl.
enableAutoSearch.value = true;
if (cfg.value.autoSearchSimilar) {
findSimilar(pop.selectedPost);
}
}
}
@ -272,11 +284,14 @@ async function loadTagCounts() {
async function fetchPostInfo() {
for (const post of pop.posts) {
if (!post.contentSize) {
if (!post.contentSize || post.extraContentUrl) {
// We are missing cookies/etc which means that the request might fail.
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
try {
const res = await fetch(post.contentUrl, { method: "HEAD" });
const contentUrl = post.extraContentUrl ?? post.contentUrl;
// TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup.
const res = await fetch(contentUrl, { method: "HEAD" });
const size = res.headers.get("Content-Length");
const type = res.headers.get("Content-Type");
@ -286,8 +301,14 @@ async function fetchPostInfo() {
const [_main, sub] = type.split("/");
if (sub) post.contentSubType = sub.toUpperCase();
}
} catch {
// TODO: Log errors.
// Resolve extraContentUrl redirects.
if (post.extraContentUrl && res.url != post.contentUrl) {
post.contentUrl = res.url;
}
} catch (ex) {
// TODO: Maybe display an error in the UI.
console.error(ex);
}
}
}