Update neo-scraper and add experimental support for e-hentai
parent
96466b9199
commit
c0ac677794
1
.npmrc
1
.npmrc
|
|
@ -1,2 +1,3 @@
|
||||||
shamefully-hoist=true
|
shamefully-hoist=true
|
||||||
auto-install-peers=true
|
auto-install-peers=true
|
||||||
|
strict-peer-dependencies=false
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@
|
||||||
"deepmerge": "^4.3.1",
|
"deepmerge": "^4.3.1",
|
||||||
"lodash": "^4.17.21",
|
"lodash": "^4.17.21",
|
||||||
"markdown-it": "^13.0.1",
|
"markdown-it": "^13.0.1",
|
||||||
"neo-scraper": "^0.5.8",
|
"neo-scraper": "^0.6.0",
|
||||||
"normalize.css": "^8.0.1",
|
"normalize.css": "^8.0.1",
|
||||||
"pinia": "^2.0.36",
|
"pinia": "^2.0.36",
|
||||||
"vue-router": "4"
|
"vue-router": "4"
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ specifiers:
|
||||||
kolorist: ^1.6.0
|
kolorist: ^1.6.0
|
||||||
lodash: ^4.17.21
|
lodash: ^4.17.21
|
||||||
markdown-it: ^13.0.1
|
markdown-it: ^13.0.1
|
||||||
neo-scraper: ^0.5.8
|
neo-scraper: ^0.6.0
|
||||||
normalize.css: ^8.0.1
|
normalize.css: ^8.0.1
|
||||||
npm-run-all: ^4.1.5
|
npm-run-all: ^4.1.5
|
||||||
pinia: ^2.0.36
|
pinia: ^2.0.36
|
||||||
|
|
@ -55,7 +55,7 @@ dependencies:
|
||||||
deepmerge: 4.3.1
|
deepmerge: 4.3.1
|
||||||
lodash: 4.17.21
|
lodash: 4.17.21
|
||||||
markdown-it: 13.0.1
|
markdown-it: 13.0.1
|
||||||
neo-scraper: 0.5.8
|
neo-scraper: 0.6.0
|
||||||
normalize.css: 8.0.1
|
normalize.css: 8.0.1
|
||||||
pinia: 2.0.36_hmuptsblhheur2tugfgucj7gc4
|
pinia: 2.0.36_hmuptsblhheur2tugfgucj7gc4
|
||||||
vue-router: 4.1.6_vue@3.2.47
|
vue-router: 4.1.6_vue@3.2.47
|
||||||
|
|
@ -3960,6 +3960,11 @@ packages:
|
||||||
nopt: 6.0.0
|
nopt: 6.0.0
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
|
/js-cookie/3.0.5:
|
||||||
|
resolution: {integrity: sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==}
|
||||||
|
engines: {node: '>=14'}
|
||||||
|
dev: false
|
||||||
|
|
||||||
/js-sdsl/4.4.0:
|
/js-sdsl/4.4.0:
|
||||||
resolution: {integrity: sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==}
|
resolution: {integrity: sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
@ -4590,9 +4595,10 @@ packages:
|
||||||
dev: true
|
dev: true
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
/neo-scraper/0.5.8:
|
/neo-scraper/0.6.0:
|
||||||
resolution: {integrity: sha512-81SUT7Fo1o7z6HzbhT+GdGY1pAo+1E+s2WmFm08WDe02n9crpRy4cOFYdTtINa/W224YtMvKVC23YD6eLAlayQ==}
|
resolution: {integrity: sha512-ytvdnj4NEaCw9KS+Zgg1lv5t/8l5Sq0jsDVW4OykvWQs4W35skcEnYQyL/LnKCmQ7aU5ZywIeVllvh8LiAZgcg==}
|
||||||
dependencies:
|
dependencies:
|
||||||
|
js-cookie: 3.0.5
|
||||||
turndown: 7.1.2
|
turndown: 7.1.2
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import {
|
||||||
SetPostUploadInfoData,
|
SetPostUploadInfoData,
|
||||||
SetExactPostId,
|
SetExactPostId,
|
||||||
PostUpdateCommandData,
|
PostUpdateCommandData,
|
||||||
|
FetchCommandData,
|
||||||
} from "~/models";
|
} from "~/models";
|
||||||
import { PostAlreadyUploadedError } from "~/api/models";
|
import { PostAlreadyUploadedError } from "~/api/models";
|
||||||
import SzurubooruApi from "~/api";
|
import SzurubooruApi from "~/api";
|
||||||
|
|
@ -115,11 +116,11 @@ async function updatePost(data: PostUpdateCommandData) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const szuru = SzurubooruApi.createFromConfig(data.selectedSite);
|
const szuru = SzurubooruApi.createFromConfig(data.selectedSite);
|
||||||
|
|
||||||
pushInfo();
|
pushInfo();
|
||||||
|
|
||||||
await szuru.updatePost(data.postId, data.updateRequest);
|
await szuru.updatePost(data.postId, data.updateRequest);
|
||||||
|
|
||||||
info.state = "uploaded";
|
info.state = "uploaded";
|
||||||
pushInfo();
|
pushInfo();
|
||||||
} catch (ex: any) {
|
} catch (ex: any) {
|
||||||
|
|
@ -129,6 +130,15 @@ async function updatePost(data: PostUpdateCommandData) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Executes fetch in the background page. This allows us to do "forbidden" stuff, like ignoring CORS headers.
|
||||||
|
* @param data
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
async function executeFetch(data: FetchCommandData) {
|
||||||
|
return await fetch(data.url, data.options);
|
||||||
|
}
|
||||||
|
|
||||||
async function messageHandler(cmd: BrowserCommand): Promise<any> {
|
async function messageHandler(cmd: BrowserCommand): Promise<any> {
|
||||||
console.log("Background received message:");
|
console.log("Background received message:");
|
||||||
console.dir(cmd);
|
console.dir(cmd);
|
||||||
|
|
@ -138,6 +148,8 @@ async function messageHandler(cmd: BrowserCommand): Promise<any> {
|
||||||
return uploadPost(cmd.data);
|
return uploadPost(cmd.data);
|
||||||
case "update_post":
|
case "update_post":
|
||||||
return updatePost(cmd.data);
|
return updatePost(cmd.data);
|
||||||
|
case "fetch":
|
||||||
|
return executeFetch(cmd.data);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,7 @@ export class ScrapedPostDetails {
|
||||||
tags: TagDetails[] = [];
|
tags: TagDetails[] = [];
|
||||||
notes: ScrapedNote[];
|
notes: ScrapedNote[];
|
||||||
contentUrl: string;
|
contentUrl: string;
|
||||||
|
extraContentUrl: string | undefined;
|
||||||
contentSize: number | undefined;
|
contentSize: number | undefined;
|
||||||
pageUrl: string;
|
pageUrl: string;
|
||||||
contentType: ContentType;
|
contentType: ContentType;
|
||||||
|
|
@ -59,6 +60,7 @@ export class ScrapedPostDetails {
|
||||||
|
|
||||||
constructor(post: ScrapedPost) {
|
constructor(post: ScrapedPost) {
|
||||||
this.contentUrl = post.contentUrl;
|
this.contentUrl = post.contentUrl;
|
||||||
|
this.extraContentUrl = post.extraContentUrl;
|
||||||
// this.contentSize = post.contentSize;
|
// this.contentSize = post.contentSize;
|
||||||
this.pageUrl = post.pageUrl;
|
this.pageUrl = post.pageUrl;
|
||||||
this.contentType = post.contentType;
|
this.contentType = post.contentType;
|
||||||
|
|
@ -104,7 +106,8 @@ export type BrowserCommandName =
|
||||||
| "set_post_upload_info"
|
| "set_post_upload_info"
|
||||||
| "set_exact_post_id"
|
| "set_exact_post_id"
|
||||||
| "update_post"
|
| "update_post"
|
||||||
| "set_post_update_info";
|
| "set_post_update_info"
|
||||||
|
| "fetch";
|
||||||
|
|
||||||
export class BrowserCommand<T = any> {
|
export class BrowserCommand<T = any> {
|
||||||
name: BrowserCommandName;
|
name: BrowserCommandName;
|
||||||
|
|
@ -139,3 +142,7 @@ export class PostUpdateCommandData {
|
||||||
public readonly selectedSite: SzuruSiteConfig
|
public readonly selectedSite: SzuruSiteConfig
|
||||||
) {}
|
) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class FetchCommandData {
|
||||||
|
constructor(public readonly url: string, public readonly options: RequestInit | undefined = undefined) {}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ import SzurubooruApi from "~/api";
|
||||||
|
|
||||||
const pop = usePopupStore();
|
const pop = usePopupStore();
|
||||||
const isSearchingForSimilarPosts = ref<number>(0);
|
const isSearchingForSimilarPosts = ref<number>(0);
|
||||||
|
const enableAutoSearch = ref(true);
|
||||||
|
|
||||||
const selectedSite = computed(() => {
|
const selectedSite = computed(() => {
|
||||||
if (cfg.value.selectedSiteId) {
|
if (cfg.value.selectedSiteId) {
|
||||||
|
|
@ -46,7 +47,7 @@ watch(
|
||||||
() => pop.selectedPostId,
|
() => pop.selectedPostId,
|
||||||
(value) => {
|
(value) => {
|
||||||
// Call findSimilar if wanted.
|
// Call findSimilar if wanted.
|
||||||
if (cfg.value.autoSearchSimilar) {
|
if (cfg.value.autoSearchSimilar && enableAutoSearch.value) {
|
||||||
let selectedPost = pop.posts.find((x) => x.id == value);
|
let selectedPost = pop.posts.find((x) => x.id == value);
|
||||||
if (selectedPost) findSimilar(selectedPost);
|
if (selectedPost) findSimilar(selectedPost);
|
||||||
}
|
}
|
||||||
|
|
@ -120,13 +121,24 @@ async function grabPost() {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (pop.posts.length > 0) {
|
if (pop.posts.length > 0) {
|
||||||
|
// Hack so that findSimilar does not trigger when we set selectedPostId, due to the watch(...).
|
||||||
|
// This because we might change the contentUrl in fetchPostInfo, after which we'd need to call findSimilar again.
|
||||||
|
// In which case it would be better to just call it once.
|
||||||
|
enableAutoSearch.value = false;
|
||||||
pop.selectedPostId = pop.posts[0].id;
|
pop.selectedPostId = pop.posts[0].id;
|
||||||
|
|
||||||
if (cfg.value.loadTagCounts) {
|
if (cfg.value.loadTagCounts) {
|
||||||
loadTagCounts();
|
loadTagCounts();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (cfg.value.fetchPostInfo) {
|
if (cfg.value.fetchPostInfo) {
|
||||||
fetchPostInfo();
|
await fetchPostInfo();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl.
|
||||||
|
enableAutoSearch.value = true;
|
||||||
|
if (cfg.value.autoSearchSimilar) {
|
||||||
|
findSimilar(pop.selectedPost);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -272,11 +284,14 @@ async function loadTagCounts() {
|
||||||
|
|
||||||
async function fetchPostInfo() {
|
async function fetchPostInfo() {
|
||||||
for (const post of pop.posts) {
|
for (const post of pop.posts) {
|
||||||
if (!post.contentSize) {
|
if (!post.contentSize || post.extraContentUrl) {
|
||||||
// We are missing cookies/etc which means that the request might fail.
|
// We are missing cookies/etc which means that the request might fail.
|
||||||
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
|
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
|
||||||
try {
|
try {
|
||||||
const res = await fetch(post.contentUrl, { method: "HEAD" });
|
const contentUrl = post.extraContentUrl ?? post.contentUrl;
|
||||||
|
|
||||||
|
// TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup.
|
||||||
|
const res = await fetch(contentUrl, { method: "HEAD" });
|
||||||
const size = res.headers.get("Content-Length");
|
const size = res.headers.get("Content-Length");
|
||||||
const type = res.headers.get("Content-Type");
|
const type = res.headers.get("Content-Type");
|
||||||
|
|
||||||
|
|
@ -286,8 +301,14 @@ async function fetchPostInfo() {
|
||||||
const [_main, sub] = type.split("/");
|
const [_main, sub] = type.split("/");
|
||||||
if (sub) post.contentSubType = sub.toUpperCase();
|
if (sub) post.contentSubType = sub.toUpperCase();
|
||||||
}
|
}
|
||||||
} catch {
|
|
||||||
// TODO: Log errors.
|
// Resolve extraContentUrl redirects.
|
||||||
|
if (post.extraContentUrl && res.url != post.contentUrl) {
|
||||||
|
post.contentUrl = res.url;
|
||||||
|
}
|
||||||
|
} catch (ex) {
|
||||||
|
// TODO: Maybe display an error in the UI.
|
||||||
|
console.error(ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue