Improved workaround for sites with hotlink protection or which require authentication
parent
3d9586147b
commit
9843f0b641
|
|
@ -27,12 +27,12 @@ runs:
|
||||||
name: Install pnpm
|
name: Install pnpm
|
||||||
id: pnpm-install
|
id: pnpm-install
|
||||||
with:
|
with:
|
||||||
version: 7
|
version: 9
|
||||||
run_install: false
|
run_install: false
|
||||||
|
|
||||||
- uses: actions/setup-node@v3
|
- uses: actions/setup-node@v3
|
||||||
with:
|
with:
|
||||||
node-version: "16"
|
node-version: "20"
|
||||||
cache: "pnpm"
|
cache: "pnpm"
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,6 @@
|
||||||
"description": "Import media from various sites into szurubooru.",
|
"description": "Import media from various sites into szurubooru.",
|
||||||
"version": "0.0.0",
|
"version": "0.0.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"packageManager": "pnpm@7.8.0",
|
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "npm run clear && cross-env NODE_ENV=development run-p dev:*",
|
"dev": "npm run clear && cross-env NODE_ENV=development run-p dev:*",
|
||||||
"dev:prepare": "esno scripts/prepare.ts",
|
"dev:prepare": "esno scripts/prepare.ts",
|
||||||
|
|
@ -33,7 +32,7 @@
|
||||||
"deepmerge": "^4.3.1",
|
"deepmerge": "^4.3.1",
|
||||||
"lodash": "^4.17.21",
|
"lodash": "^4.17.21",
|
||||||
"markdown-it": "^14.0.0",
|
"markdown-it": "^14.0.0",
|
||||||
"neo-scraper": "^0.7.5",
|
"neo-scraper": "^0.8.0",
|
||||||
"normalize.css": "^8.0.1",
|
"normalize.css": "^8.0.1",
|
||||||
"pinia": "^2.1.7",
|
"pinia": "^2.1.7",
|
||||||
"primeflex": "^3.3.1",
|
"primeflex": "^3.3.1",
|
||||||
|
|
|
||||||
7794
pnpm-lock.yaml
7794
pnpm-lock.yaml
File diff suppressed because it is too large
Load Diff
|
|
@ -16,6 +16,7 @@ import {
|
||||||
UpdatePoolRequest,
|
UpdatePoolRequest,
|
||||||
} from "./models";
|
} from "./models";
|
||||||
import { ScrapedPostDetails, SzuruSiteConfig } from "~/models";
|
import { ScrapedPostDetails, SzuruSiteConfig } from "~/models";
|
||||||
|
import { UploadMode } from "neo-scraper";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A 1:1 wrapper around the szurubooru API.
|
* A 1:1 wrapper around the szurubooru API.
|
||||||
|
|
@ -147,11 +148,11 @@ export default class SzurubooruApi {
|
||||||
return (await this.apiPost("posts/reverse-search", obj)).data;
|
return (await this.apiPost("posts/reverse-search", obj)).data;
|
||||||
}
|
}
|
||||||
|
|
||||||
async uploadTempFile(contentUrl: string): Promise<TemporaryFileUploadResult> {
|
async uploadTempFile(contentUrl: string, uploadMode: UploadMode): Promise<TemporaryFileUploadResult> {
|
||||||
// HACK: For some sources we need to download the image to the client and then upload it to szurubooru.
|
// For some sources we need to download the image to the client and then upload it to szurubooru.
|
||||||
// We can't just pass the contentUrl because that would trigger the bot/hotlink protection.
|
// We can't just pass the contentUrl because that would trigger the bot/hotlink protection.
|
||||||
|
|
||||||
if (contentUrl.indexOf("donmai.us") != -1) {
|
if (uploadMode == "content") {
|
||||||
console.log("Upload from content");
|
console.log("Upload from content");
|
||||||
return this.uploadTempFileFromContent(contentUrl);
|
return this.uploadTempFileFromContent(contentUrl);
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag } from "neo-scraper";
|
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag, UploadMode } from "neo-scraper";
|
||||||
import { MicroTag, Pool, Tag, UpdatePostRequest } from "~/api/models";
|
import { MicroTag, Pool, Tag, UpdatePostRequest } from "~/api/models";
|
||||||
|
|
||||||
export class TagDetails {
|
export class TagDetails {
|
||||||
|
|
@ -77,6 +77,7 @@ export class ScrapedPostDetails {
|
||||||
referrer?: string;
|
referrer?: string;
|
||||||
resolution?: [number, number];
|
resolution?: [number, number];
|
||||||
instanceSpecificData: MappedInstanceSpecificData = {};
|
instanceSpecificData: MappedInstanceSpecificData = {};
|
||||||
|
uploadMode: UploadMode = "url";
|
||||||
|
|
||||||
constructor(post: ScrapedPost) {
|
constructor(post: ScrapedPost) {
|
||||||
this.contentUrl = post.contentUrl;
|
this.contentUrl = post.contentUrl;
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { useDark } from "@vueuse/core";
|
import { useDark } from "@vueuse/core";
|
||||||
import { cloneDeep } from "lodash";
|
import { cloneDeep } from "lodash";
|
||||||
import { ScrapeResults } from "neo-scraper";
|
import { NeoScraper, ScrapeResults } from "neo-scraper";
|
||||||
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
|
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
|
||||||
import {
|
import {
|
||||||
BrowserCommand,
|
BrowserCommand,
|
||||||
|
|
@ -92,14 +92,18 @@ async function grabPost() {
|
||||||
const x = await browser.tabs.sendMessage(activeTabId, new BrowserCommand("grab_post"));
|
const x = await browser.tabs.sendMessage(activeTabId, new BrowserCommand("grab_post"));
|
||||||
// We need to create a new ScrapeResults object and fill it with our data, because the get_posts()
|
// We need to create a new ScrapeResults object and fill it with our data, because the get_posts()
|
||||||
// method gets 'lost' when sent from the contentscript to the popup (it gets JSON.stringified and any prototype defines are lost there)
|
// method gets 'lost' when sent from the contentscript to the popup (it gets JSON.stringified and any prototype defines are lost there)
|
||||||
const res = Object.assign(new ScrapeResults(), x);
|
const res: ScrapeResults = Object.assign(new ScrapeResults(), x);
|
||||||
|
|
||||||
console.dir(res);
|
console.dir(res);
|
||||||
|
|
||||||
// Clear current posts array
|
// Clear current posts array
|
||||||
pop.posts.splice(0);
|
pop.posts.splice(0);
|
||||||
|
|
||||||
|
const scraper = new NeoScraper();
|
||||||
|
|
||||||
for (const result of res.results) {
|
for (const result of res.results) {
|
||||||
|
const engine = scraper.engines.find((x) => x.name == result.engine);
|
||||||
|
|
||||||
for (const i in result.posts) {
|
for (const i in result.posts) {
|
||||||
const vm = new ScrapedPostDetails(result.posts[i]);
|
const vm = new ScrapedPostDetails(result.posts[i]);
|
||||||
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
|
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
|
||||||
|
|
@ -121,6 +125,10 @@ async function grabPost() {
|
||||||
vm.instanceSpecificData[site.id] = {};
|
vm.instanceSpecificData[site.id] = {};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (engine) {
|
||||||
|
vm.uploadMode = engine.uploadMode;
|
||||||
|
}
|
||||||
|
|
||||||
pop.posts.push(vm);
|
pop.posts.push(vm);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -160,7 +168,13 @@ async function upload() {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const post: any = cloneDeep(pop.selectedPost);
|
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost);
|
||||||
|
|
||||||
|
// uploadMode "content" requires a content token to work. So ensure it is set.
|
||||||
|
if (post.uploadMode == "content") {
|
||||||
|
await ensurePostHasContentToken(post);
|
||||||
|
}
|
||||||
|
|
||||||
const cmdData = new PostUploadCommandData(post, <SzuruSiteConfig>cloneDeep(selectedSite.value));
|
const cmdData = new PostUploadCommandData(post, <SzuruSiteConfig>cloneDeep(selectedSite.value));
|
||||||
const cmd = new BrowserCommand("upload_post", cmdData);
|
const cmd = new BrowserCommand("upload_post", cmdData);
|
||||||
await browser.runtime.sendMessage(cmd);
|
await browser.runtime.sendMessage(cmd);
|
||||||
|
|
@ -239,6 +253,27 @@ async function clickFindSimilar() {
|
||||||
if (pop.selectedPost) return await findSimilar(pop.selectedPost);
|
if (pop.selectedPost) return await findSimilar(pop.selectedPost);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function ensurePostHasContentToken(post: ScrapedPostDetails) {
|
||||||
|
if (!szuru.value || !cfg.value.selectedSiteId) return;
|
||||||
|
|
||||||
|
const selectedInstance = toRaw(szuru.value); // Get current object, and not reactive.
|
||||||
|
let instanceSpecificData = post.instanceSpecificData[cfg.value.selectedSiteId];
|
||||||
|
|
||||||
|
if (!instanceSpecificData) {
|
||||||
|
console.error("instanceSpecificData is undefined. This should never happen!");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
let tmpRes = await selectedInstance.uploadTempFile(post.contentUrl, post.uploadMode);
|
||||||
|
// Save contentToken in PostViewModel so that we can reuse it when creating/uploading the post.
|
||||||
|
instanceSpecificData.contentToken = tmpRes.token;
|
||||||
|
} catch (ex) {
|
||||||
|
instanceSpecificData.genericError = "Couldn't upload content. " + getErrorMessage(ex);
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function findSimilar(post: ScrapedPostDetails | undefined) {
|
async function findSimilar(post: ScrapedPostDetails | undefined) {
|
||||||
if (!post || !szuru.value || !cfg.value.selectedSiteId) return;
|
if (!post || !szuru.value || !cfg.value.selectedSiteId) return;
|
||||||
|
|
||||||
|
|
@ -252,23 +287,14 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Don't load this again if it is already loaded.
|
// Don't load this again if it is already loaded.
|
||||||
if (instanceSpecificData?.reverseSearchResult) return;
|
if (instanceSpecificData.reverseSearchResult) return;
|
||||||
|
|
||||||
isSearchingForSimilarPosts.value++;
|
isSearchingForSimilarPosts.value++;
|
||||||
|
|
||||||
|
await ensurePostHasContentToken(post);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
let res: ImageSearchResult | undefined;
|
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
|
||||||
|
|
||||||
if (cfg.value.useContentTokens) {
|
|
||||||
let tmpRes = await selectedInstance.uploadTempFile(post.contentUrl);
|
|
||||||
// TODO: Error handling?
|
|
||||||
// Save contentToken in PostViewModel so that we can reuse it when creating/uploading the post.
|
|
||||||
instanceSpecificData.contentToken = tmpRes.token;
|
|
||||||
|
|
||||||
res = await selectedInstance.reverseSearchToken(tmpRes.token);
|
|
||||||
} else {
|
|
||||||
res = await selectedInstance.reverseSearch(post.contentUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
instanceSpecificData.reverseSearchResult = {
|
instanceSpecificData.reverseSearchResult = {
|
||||||
exactPostId: res.exactPost?.id,
|
exactPostId: res.exactPost?.id,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue