Improved workaround for sites with hotlink protection or which require authentication
parent
3d9586147b
commit
9843f0b641
|
|
@ -27,12 +27,12 @@ runs:
|
|||
name: Install pnpm
|
||||
id: pnpm-install
|
||||
with:
|
||||
version: 7
|
||||
version: 9
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: "16"
|
||||
node-version: "20"
|
||||
cache: "pnpm"
|
||||
|
||||
- name: Install dependencies
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
"description": "Import media from various sites into szurubooru.",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@7.8.0",
|
||||
"scripts": {
|
||||
"dev": "npm run clear && cross-env NODE_ENV=development run-p dev:*",
|
||||
"dev:prepare": "esno scripts/prepare.ts",
|
||||
|
|
@ -33,7 +32,7 @@
|
|||
"deepmerge": "^4.3.1",
|
||||
"lodash": "^4.17.21",
|
||||
"markdown-it": "^14.0.0",
|
||||
"neo-scraper": "^0.7.5",
|
||||
"neo-scraper": "^0.8.0",
|
||||
"normalize.css": "^8.0.1",
|
||||
"pinia": "^2.1.7",
|
||||
"primeflex": "^3.3.1",
|
||||
|
|
|
|||
7794
pnpm-lock.yaml
7794
pnpm-lock.yaml
File diff suppressed because it is too large
Load Diff
|
|
@ -16,6 +16,7 @@ import {
|
|||
UpdatePoolRequest,
|
||||
} from "./models";
|
||||
import { ScrapedPostDetails, SzuruSiteConfig } from "~/models";
|
||||
import { UploadMode } from "neo-scraper";
|
||||
|
||||
/**
|
||||
* A 1:1 wrapper around the szurubooru API.
|
||||
|
|
@ -147,11 +148,11 @@ export default class SzurubooruApi {
|
|||
return (await this.apiPost("posts/reverse-search", obj)).data;
|
||||
}
|
||||
|
||||
async uploadTempFile(contentUrl: string): Promise<TemporaryFileUploadResult> {
|
||||
// HACK: For some sources we need to download the image to the client and then upload it to szurubooru.
|
||||
async uploadTempFile(contentUrl: string, uploadMode: UploadMode): Promise<TemporaryFileUploadResult> {
|
||||
// For some sources we need to download the image to the client and then upload it to szurubooru.
|
||||
// We can't just pass the contentUrl because that would trigger the bot/hotlink protection.
|
||||
|
||||
if (contentUrl.indexOf("donmai.us") != -1) {
|
||||
if (uploadMode == "content") {
|
||||
console.log("Upload from content");
|
||||
return this.uploadTempFileFromContent(contentUrl);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag } from "neo-scraper";
|
||||
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag, UploadMode } from "neo-scraper";
|
||||
import { MicroTag, Pool, Tag, UpdatePostRequest } from "~/api/models";
|
||||
|
||||
export class TagDetails {
|
||||
|
|
@ -77,6 +77,7 @@ export class ScrapedPostDetails {
|
|||
referrer?: string;
|
||||
resolution?: [number, number];
|
||||
instanceSpecificData: MappedInstanceSpecificData = {};
|
||||
uploadMode: UploadMode = "url";
|
||||
|
||||
constructor(post: ScrapedPost) {
|
||||
this.contentUrl = post.contentUrl;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<script setup lang="ts">
|
||||
import { useDark } from "@vueuse/core";
|
||||
import { cloneDeep } from "lodash";
|
||||
import { ScrapeResults } from "neo-scraper";
|
||||
import { NeoScraper, ScrapeResults } from "neo-scraper";
|
||||
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
|
||||
import {
|
||||
BrowserCommand,
|
||||
|
|
@ -92,14 +92,18 @@ async function grabPost() {
|
|||
const x = await browser.tabs.sendMessage(activeTabId, new BrowserCommand("grab_post"));
|
||||
// We need to create a new ScrapeResults object and fill it with our data, because the get_posts()
|
||||
// method gets 'lost' when sent from the contentscript to the popup (it gets JSON.stringified and any prototype defines are lost there)
|
||||
const res = Object.assign(new ScrapeResults(), x);
|
||||
const res: ScrapeResults = Object.assign(new ScrapeResults(), x);
|
||||
|
||||
console.dir(res);
|
||||
|
||||
// Clear current posts array
|
||||
pop.posts.splice(0);
|
||||
|
||||
const scraper = new NeoScraper();
|
||||
|
||||
for (const result of res.results) {
|
||||
const engine = scraper.engines.find((x) => x.name == result.engine);
|
||||
|
||||
for (const i in result.posts) {
|
||||
const vm = new ScrapedPostDetails(result.posts[i]);
|
||||
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
|
||||
|
|
@ -121,6 +125,10 @@ async function grabPost() {
|
|||
vm.instanceSpecificData[site.id] = {};
|
||||
}
|
||||
|
||||
if (engine) {
|
||||
vm.uploadMode = engine.uploadMode;
|
||||
}
|
||||
|
||||
pop.posts.push(vm);
|
||||
}
|
||||
}
|
||||
|
|
@ -160,7 +168,13 @@ async function upload() {
|
|||
}
|
||||
|
||||
try {
|
||||
const post: any = cloneDeep(pop.selectedPost);
|
||||
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost);
|
||||
|
||||
// uploadMode "content" requires a content token to work. So ensure it is set.
|
||||
if (post.uploadMode == "content") {
|
||||
await ensurePostHasContentToken(post);
|
||||
}
|
||||
|
||||
const cmdData = new PostUploadCommandData(post, <SzuruSiteConfig>cloneDeep(selectedSite.value));
|
||||
const cmd = new BrowserCommand("upload_post", cmdData);
|
||||
await browser.runtime.sendMessage(cmd);
|
||||
|
|
@ -239,6 +253,27 @@ async function clickFindSimilar() {
|
|||
if (pop.selectedPost) return await findSimilar(pop.selectedPost);
|
||||
}
|
||||
|
||||
async function ensurePostHasContentToken(post: ScrapedPostDetails) {
|
||||
if (!szuru.value || !cfg.value.selectedSiteId) return;
|
||||
|
||||
const selectedInstance = toRaw(szuru.value); // Get current object, and not reactive.
|
||||
let instanceSpecificData = post.instanceSpecificData[cfg.value.selectedSiteId];
|
||||
|
||||
if (!instanceSpecificData) {
|
||||
console.error("instanceSpecificData is undefined. This should never happen!");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
let tmpRes = await selectedInstance.uploadTempFile(post.contentUrl, post.uploadMode);
|
||||
// Save contentToken in PostViewModel so that we can reuse it when creating/uploading the post.
|
||||
instanceSpecificData.contentToken = tmpRes.token;
|
||||
} catch (ex) {
|
||||
instanceSpecificData.genericError = "Couldn't upload content. " + getErrorMessage(ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
async function findSimilar(post: ScrapedPostDetails | undefined) {
|
||||
if (!post || !szuru.value || !cfg.value.selectedSiteId) return;
|
||||
|
||||
|
|
@ -252,23 +287,14 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
|
|||
}
|
||||
|
||||
// Don't load this again if it is already loaded.
|
||||
if (instanceSpecificData?.reverseSearchResult) return;
|
||||
if (instanceSpecificData.reverseSearchResult) return;
|
||||
|
||||
isSearchingForSimilarPosts.value++;
|
||||
|
||||
await ensurePostHasContentToken(post);
|
||||
|
||||
try {
|
||||
let res: ImageSearchResult | undefined;
|
||||
|
||||
if (cfg.value.useContentTokens) {
|
||||
let tmpRes = await selectedInstance.uploadTempFile(post.contentUrl);
|
||||
// TODO: Error handling?
|
||||
// Save contentToken in PostViewModel so that we can reuse it when creating/uploading the post.
|
||||
instanceSpecificData.contentToken = tmpRes.token;
|
||||
|
||||
res = await selectedInstance.reverseSearchToken(tmpRes.token);
|
||||
} else {
|
||||
res = await selectedInstance.reverseSearch(post.contentUrl);
|
||||
}
|
||||
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
|
||||
|
||||
instanceSpecificData.reverseSearchResult = {
|
||||
exactPostId: res.exactPost?.id,
|
||||
|
|
|
|||
Loading…
Reference in New Issue