Improved workaround for sites with hotlink protection or which require authentication

pull/55/head v0.12.0
neobooru 2024-06-14 16:59:57 +02:00
parent 3d9586147b
commit 9843f0b641
6 changed files with 4361 additions and 3508 deletions

View File

@ -27,12 +27,12 @@ runs:
name: Install pnpm name: Install pnpm
id: pnpm-install id: pnpm-install
with: with:
version: 7 version: 9
run_install: false run_install: false
- uses: actions/setup-node@v3 - uses: actions/setup-node@v3
with: with:
node-version: "16" node-version: "20"
cache: "pnpm" cache: "pnpm"
- name: Install dependencies - name: Install dependencies

View File

@ -4,7 +4,6 @@
"description": "Import media from various sites into szurubooru.", "description": "Import media from various sites into szurubooru.",
"version": "0.0.0", "version": "0.0.0",
"private": true, "private": true,
"packageManager": "pnpm@7.8.0",
"scripts": { "scripts": {
"dev": "npm run clear && cross-env NODE_ENV=development run-p dev:*", "dev": "npm run clear && cross-env NODE_ENV=development run-p dev:*",
"dev:prepare": "esno scripts/prepare.ts", "dev:prepare": "esno scripts/prepare.ts",
@ -33,7 +32,7 @@
"deepmerge": "^4.3.1", "deepmerge": "^4.3.1",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"markdown-it": "^14.0.0", "markdown-it": "^14.0.0",
"neo-scraper": "^0.7.5", "neo-scraper": "^0.8.0",
"normalize.css": "^8.0.1", "normalize.css": "^8.0.1",
"pinia": "^2.1.7", "pinia": "^2.1.7",
"primeflex": "^3.3.1", "primeflex": "^3.3.1",

File diff suppressed because it is too large Load Diff

View File

@ -16,6 +16,7 @@ import {
UpdatePoolRequest, UpdatePoolRequest,
} from "./models"; } from "./models";
import { ScrapedPostDetails, SzuruSiteConfig } from "~/models"; import { ScrapedPostDetails, SzuruSiteConfig } from "~/models";
import { UploadMode } from "neo-scraper";
/** /**
* A 1:1 wrapper around the szurubooru API. * A 1:1 wrapper around the szurubooru API.
@ -147,11 +148,11 @@ export default class SzurubooruApi {
return (await this.apiPost("posts/reverse-search", obj)).data; return (await this.apiPost("posts/reverse-search", obj)).data;
} }
async uploadTempFile(contentUrl: string): Promise<TemporaryFileUploadResult> { async uploadTempFile(contentUrl: string, uploadMode: UploadMode): Promise<TemporaryFileUploadResult> {
// HACK: For some sources we need to download the image to the client and then upload it to szurubooru. // For some sources we need to download the image to the client and then upload it to szurubooru.
// We can't just pass the contentUrl because that would trigger the bot/hotlink protection. // We can't just pass the contentUrl because that would trigger the bot/hotlink protection.
if (contentUrl.indexOf("donmai.us") != -1) { if (uploadMode == "content") {
console.log("Upload from content"); console.log("Upload from content");
return this.uploadTempFileFromContent(contentUrl); return this.uploadTempFileFromContent(contentUrl);
} else { } else {

View File

@ -1,4 +1,4 @@
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag } from "neo-scraper"; import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag, UploadMode } from "neo-scraper";
import { MicroTag, Pool, Tag, UpdatePostRequest } from "~/api/models"; import { MicroTag, Pool, Tag, UpdatePostRequest } from "~/api/models";
export class TagDetails { export class TagDetails {
@ -77,6 +77,7 @@ export class ScrapedPostDetails {
referrer?: string; referrer?: string;
resolution?: [number, number]; resolution?: [number, number];
instanceSpecificData: MappedInstanceSpecificData = {}; instanceSpecificData: MappedInstanceSpecificData = {};
uploadMode: UploadMode = "url";
constructor(post: ScrapedPost) { constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl; this.contentUrl = post.contentUrl;

View File

@ -1,7 +1,7 @@
<script setup lang="ts"> <script setup lang="ts">
import { useDark } from "@vueuse/core"; import { useDark } from "@vueuse/core";
import { cloneDeep } from "lodash"; import { cloneDeep } from "lodash";
import { ScrapeResults } from "neo-scraper"; import { NeoScraper, ScrapeResults } from "neo-scraper";
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils"; import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
import { import {
BrowserCommand, BrowserCommand,
@ -92,14 +92,18 @@ async function grabPost() {
const x = await browser.tabs.sendMessage(activeTabId, new BrowserCommand("grab_post")); const x = await browser.tabs.sendMessage(activeTabId, new BrowserCommand("grab_post"));
// We need to create a new ScrapeResults object and fill it with our data, because the get_posts() // We need to create a new ScrapeResults object and fill it with our data, because the get_posts()
// method gets 'lost' when sent from the contentscript to the popup (it gets JSON.stringified and any prototype defines are lost there) // method gets 'lost' when sent from the contentscript to the popup (it gets JSON.stringified and any prototype defines are lost there)
const res = Object.assign(new ScrapeResults(), x); const res: ScrapeResults = Object.assign(new ScrapeResults(), x);
console.dir(res); console.dir(res);
// Clear current posts array // Clear current posts array
pop.posts.splice(0); pop.posts.splice(0);
const scraper = new NeoScraper();
for (const result of res.results) { for (const result of res.results) {
const engine = scraper.engines.find((x) => x.name == result.engine);
for (const i in result.posts) { for (const i in result.posts) {
const vm = new ScrapedPostDetails(result.posts[i]); const vm = new ScrapedPostDetails(result.posts[i]);
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required! vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
@ -121,6 +125,10 @@ async function grabPost() {
vm.instanceSpecificData[site.id] = {}; vm.instanceSpecificData[site.id] = {};
} }
if (engine) {
vm.uploadMode = engine.uploadMode;
}
pop.posts.push(vm); pop.posts.push(vm);
} }
} }
@ -160,7 +168,13 @@ async function upload() {
} }
try { try {
const post: any = cloneDeep(pop.selectedPost); const post: ScrapedPostDetails = cloneDeep(pop.selectedPost);
// uploadMode "content" requires a content token to work. So ensure it is set.
if (post.uploadMode == "content") {
await ensurePostHasContentToken(post);
}
const cmdData = new PostUploadCommandData(post, <SzuruSiteConfig>cloneDeep(selectedSite.value)); const cmdData = new PostUploadCommandData(post, <SzuruSiteConfig>cloneDeep(selectedSite.value));
const cmd = new BrowserCommand("upload_post", cmdData); const cmd = new BrowserCommand("upload_post", cmdData);
await browser.runtime.sendMessage(cmd); await browser.runtime.sendMessage(cmd);
@ -239,6 +253,27 @@ async function clickFindSimilar() {
if (pop.selectedPost) return await findSimilar(pop.selectedPost); if (pop.selectedPost) return await findSimilar(pop.selectedPost);
} }
async function ensurePostHasContentToken(post: ScrapedPostDetails) {
if (!szuru.value || !cfg.value.selectedSiteId) return;
const selectedInstance = toRaw(szuru.value); // Get current object, and not reactive.
let instanceSpecificData = post.instanceSpecificData[cfg.value.selectedSiteId];
if (!instanceSpecificData) {
console.error("instanceSpecificData is undefined. This should never happen!");
return;
}
try {
let tmpRes = await selectedInstance.uploadTempFile(post.contentUrl, post.uploadMode);
// Save contentToken in PostViewModel so that we can reuse it when creating/uploading the post.
instanceSpecificData.contentToken = tmpRes.token;
} catch (ex) {
instanceSpecificData.genericError = "Couldn't upload content. " + getErrorMessage(ex);
throw ex;
}
}
async function findSimilar(post: ScrapedPostDetails | undefined) { async function findSimilar(post: ScrapedPostDetails | undefined) {
if (!post || !szuru.value || !cfg.value.selectedSiteId) return; if (!post || !szuru.value || !cfg.value.selectedSiteId) return;
@ -252,23 +287,14 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
} }
// Don't load this again if it is already loaded. // Don't load this again if it is already loaded.
if (instanceSpecificData?.reverseSearchResult) return; if (instanceSpecificData.reverseSearchResult) return;
isSearchingForSimilarPosts.value++; isSearchingForSimilarPosts.value++;
await ensurePostHasContentToken(post);
try { try {
let res: ImageSearchResult | undefined; const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
if (cfg.value.useContentTokens) {
let tmpRes = await selectedInstance.uploadTempFile(post.contentUrl);
// TODO: Error handling?
// Save contentToken in PostViewModel so that we can reuse it when creating/uploading the post.
instanceSpecificData.contentToken = tmpRes.token;
res = await selectedInstance.reverseSearchToken(tmpRes.token);
} else {
res = await selectedInstance.reverseSearch(post.contentUrl);
}
instanceSpecificData.reverseSearchResult = { instanceSpecificData.reverseSearchResult = {
exactPostId: res.exactPost?.id, exactPostId: res.exactPost?.id,