Improved workaround for sites with hotlink protection or which require authentication

pull/55/head v0.12.0
neobooru 2024-06-14 16:59:57 +02:00
parent 3d9586147b
commit 9843f0b641
6 changed files with 4361 additions and 3508 deletions

View File

@ -27,12 +27,12 @@ runs:
name: Install pnpm
id: pnpm-install
with:
version: 7
version: 9
run_install: false
- uses: actions/setup-node@v3
with:
node-version: "16"
node-version: "20"
cache: "pnpm"
- name: Install dependencies

View File

@ -4,7 +4,6 @@
"description": "Import media from various sites into szurubooru.",
"version": "0.0.0",
"private": true,
"packageManager": "pnpm@7.8.0",
"scripts": {
"dev": "npm run clear && cross-env NODE_ENV=development run-p dev:*",
"dev:prepare": "esno scripts/prepare.ts",
@ -33,7 +32,7 @@
"deepmerge": "^4.3.1",
"lodash": "^4.17.21",
"markdown-it": "^14.0.0",
"neo-scraper": "^0.7.5",
"neo-scraper": "^0.8.0",
"normalize.css": "^8.0.1",
"pinia": "^2.1.7",
"primeflex": "^3.3.1",

File diff suppressed because it is too large Load Diff

View File

@ -16,6 +16,7 @@ import {
UpdatePoolRequest,
} from "./models";
import { ScrapedPostDetails, SzuruSiteConfig } from "~/models";
import { UploadMode } from "neo-scraper";
/**
* A 1:1 wrapper around the szurubooru API.
@ -147,11 +148,11 @@ export default class SzurubooruApi {
return (await this.apiPost("posts/reverse-search", obj)).data;
}
async uploadTempFile(contentUrl: string): Promise<TemporaryFileUploadResult> {
// HACK: For some sources we need to download the image to the client and then upload it to szurubooru.
async uploadTempFile(contentUrl: string, uploadMode: UploadMode): Promise<TemporaryFileUploadResult> {
// For some sources we need to download the image to the client and then upload it to szurubooru.
// We can't just pass the contentUrl because that would trigger the bot/hotlink protection.
if (contentUrl.indexOf("donmai.us") != -1) {
if (uploadMode == "content") {
console.log("Upload from content");
return this.uploadTempFileFromContent(contentUrl);
} else {

View File

@ -1,4 +1,4 @@
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag } from "neo-scraper";
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag, UploadMode } from "neo-scraper";
import { MicroTag, Pool, Tag, UpdatePostRequest } from "~/api/models";
export class TagDetails {
@ -77,6 +77,7 @@ export class ScrapedPostDetails {
referrer?: string;
resolution?: [number, number];
instanceSpecificData: MappedInstanceSpecificData = {};
uploadMode: UploadMode = "url";
constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl;

View File

@ -1,7 +1,7 @@
<script setup lang="ts">
import { useDark } from "@vueuse/core";
import { cloneDeep } from "lodash";
import { ScrapeResults } from "neo-scraper";
import { NeoScraper, ScrapeResults } from "neo-scraper";
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
import {
BrowserCommand,
@ -92,14 +92,18 @@ async function grabPost() {
const x = await browser.tabs.sendMessage(activeTabId, new BrowserCommand("grab_post"));
// We need to create a new ScrapeResults object and fill it with our data, because the get_posts()
// method gets 'lost' when sent from the contentscript to the popup (it gets JSON.stringified and any prototype defines are lost there)
const res = Object.assign(new ScrapeResults(), x);
const res: ScrapeResults = Object.assign(new ScrapeResults(), x);
console.dir(res);
// Clear current posts array
pop.posts.splice(0);
const scraper = new NeoScraper();
for (const result of res.results) {
const engine = scraper.engines.find((x) => x.name == result.engine);
for (const i in result.posts) {
const vm = new ScrapedPostDetails(result.posts[i]);
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
@ -121,6 +125,10 @@ async function grabPost() {
vm.instanceSpecificData[site.id] = {};
}
if (engine) {
vm.uploadMode = engine.uploadMode;
}
pop.posts.push(vm);
}
}
@ -160,7 +168,13 @@ async function upload() {
}
try {
const post: any = cloneDeep(pop.selectedPost);
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost);
// uploadMode "content" requires a content token to work. So ensure it is set.
if (post.uploadMode == "content") {
await ensurePostHasContentToken(post);
}
const cmdData = new PostUploadCommandData(post, <SzuruSiteConfig>cloneDeep(selectedSite.value));
const cmd = new BrowserCommand("upload_post", cmdData);
await browser.runtime.sendMessage(cmd);
@ -239,6 +253,27 @@ async function clickFindSimilar() {
if (pop.selectedPost) return await findSimilar(pop.selectedPost);
}
async function ensurePostHasContentToken(post: ScrapedPostDetails) {
if (!szuru.value || !cfg.value.selectedSiteId) return;
const selectedInstance = toRaw(szuru.value); // Get current object, and not reactive.
let instanceSpecificData = post.instanceSpecificData[cfg.value.selectedSiteId];
if (!instanceSpecificData) {
console.error("instanceSpecificData is undefined. This should never happen!");
return;
}
try {
let tmpRes = await selectedInstance.uploadTempFile(post.contentUrl, post.uploadMode);
// Save contentToken in PostViewModel so that we can reuse it when creating/uploading the post.
instanceSpecificData.contentToken = tmpRes.token;
} catch (ex) {
instanceSpecificData.genericError = "Couldn't upload content. " + getErrorMessage(ex);
throw ex;
}
}
async function findSimilar(post: ScrapedPostDetails | undefined) {
if (!post || !szuru.value || !cfg.value.selectedSiteId) return;
@ -252,23 +287,14 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
}
// Don't load this again if it is already loaded.
if (instanceSpecificData?.reverseSearchResult) return;
if (instanceSpecificData.reverseSearchResult) return;
isSearchingForSimilarPosts.value++;
await ensurePostHasContentToken(post);
try {
let res: ImageSearchResult | undefined;
if (cfg.value.useContentTokens) {
let tmpRes = await selectedInstance.uploadTempFile(post.contentUrl);
// TODO: Error handling?
// Save contentToken in PostViewModel so that we can reuse it when creating/uploading the post.
instanceSpecificData.contentToken = tmpRes.token;
res = await selectedInstance.reverseSearchToken(tmpRes.token);
} else {
res = await selectedInstance.reverseSearch(post.contentUrl);
}
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
instanceSpecificData.reverseSearchResult = {
exactPostId: res.exactPost?.id,