GlobalGit3
parent
4254c5f5d8
commit
8b89e7b527
82
README.md
82
README.md
|
|
@ -1,24 +1,66 @@
|
||||||
# neo-scraper
|
# SzuruChrome
|
||||||
|
|
||||||
Scrape a booru-like post object from the DOM.
|
Chrome/Firefox extension which can import media from various sites into a [szurubooru](https://github.com/rr-/szurubooru) instance.
|
||||||
|
|
||||||
## Supported sites
|
## Supported sites
|
||||||
|
|
||||||
| Engine | Sites | Features | Notes |
|
| Engine | Sites | Features | Notes |
|
||||||
| -------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
|
| -------------- | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------------------------- |
|
||||||
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
|
||||||
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
|
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
|
||||||
| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | |
|
| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | |
|
||||||
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | |
|
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | |
|
||||||
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
|
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
|
||||||
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
|
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
|
||||||
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | |
|
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | |
|
||||||
| pixiv | pixiv.net, www.pixiv.net | content | |
|
| philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, tags, source | |
|
||||||
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
|
| pixiv | pixiv.net, www.pixiv.net | content | |
|
||||||
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
|
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
|
||||||
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
|
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
|
||||||
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
|
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | Hotlink protection currently prevents szurubooru from importing from this site. |
|
||||||
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
|
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
|
||||||
| twitter | twitter.com, mobile.twitter.com | content | |
|
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
|
||||||
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
|
| twitter | twitter.com, mobile.twitter.com | content | |
|
||||||
| fallback | | content | Tries to find the largest image or video on the current page. |
|
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
|
||||||
|
| fallback | | content | Tries to find the largest image or video on the current page. |
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
For development see [build/debug](#Builddebug).
|
||||||
|
|
||||||
|
### Firefox
|
||||||
|
|
||||||
|
Just download the .xpi from [release tab](https://github.com/neobooru/SzuruChrome/releases) and install it.
|
||||||
|
|
||||||
|
### Chrome
|
||||||
|
|
||||||
|
I can't create a valid .crx file, which means that you'll have to use [Chrome's "Load unpacked extension"](https://developer.chrome.com/docs/extensions/mv3/getstarted/#unpacked) feature.
|
||||||
|
You can download a .zip containing the needed files from the [release tab](https://github.com/neobooru/SzuruChrome/releases). Check [this stackoverflow answer](https://stackoverflow.com/a/38011386) if you want to get rid of the developer extension popup.
|
||||||
|
|
||||||
|
## Build/debug
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Install node modules
|
||||||
|
pnpm install
|
||||||
|
|
||||||
|
# Live dev build (watch)
|
||||||
|
pnpm dev
|
||||||
|
|
||||||
|
# Production build
|
||||||
|
pnpm build
|
||||||
|
```
|
||||||
|
|
||||||
|
After this you can load them in your browser.
|
||||||
|
Chrome: `chrome://extensions/` → "Load unpacked" → Select `./dist` folder
|
||||||
|
Firefox: `about:debugging` → "Load Temporary Add-on..." → Select any file in the `./dist` folder (it will load all files)
|
||||||
|
|
||||||
|
See [the developer docs](./docs/DEVELOPER.md) for more information.
|
||||||
|
|
||||||
|
## Planned features
|
||||||
|
|
||||||
|
These are some things I might work on. Though honestly I don't expect this extension to change much anytime soon.
|
||||||
|
|
||||||
|
- Finishing the initial version of this add-on (see visual design doc for all features)
|
||||||
|
- Right click any image on a page to import it
|
||||||
|
- Configurable list of tags to always ignore (e.g. tagme, highres)
|
||||||
|
- [low priority] Import artist commentary
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ export default class NeoScraper {
|
||||||
new engines.Gelbooru(),
|
new engines.Gelbooru(),
|
||||||
new engines.Inkbunny(),
|
new engines.Inkbunny(),
|
||||||
new engines.Moebooru(),
|
new engines.Moebooru(),
|
||||||
|
new engines.Philomena(),
|
||||||
new engines.Pixiv(),
|
new engines.Pixiv(),
|
||||||
new engines.Reddit(),
|
new engines.Reddit(),
|
||||||
new engines.Rule34us(),
|
new engines.Rule34us(),
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
import { scrapeUrl } from "../test/engine.js";
|
||||||
|
|
||||||
|
test("get derpibooru.org post 380315", async () => {
|
||||||
|
const pageUrl = "https://derpibooru.org/images/3153524";
|
||||||
|
const res = await scrapeUrl(pageUrl);
|
||||||
|
expect(res.posts.length).toBe(1);
|
||||||
|
expect(res.posts[0]).toHaveTag("safe");
|
||||||
|
expect(res.posts[0]).toHaveTag("yay");
|
||||||
|
expect(res.posts[0]).toHaveTag("flutteryay");
|
||||||
|
expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640");
|
||||||
|
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||||
|
expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg");
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
|
||||||
|
import { TagCategory } from "../BooruTypes.js";
|
||||||
|
|
||||||
|
export default class Philomena extends ScrapeEngineBase {
|
||||||
|
name = "Philomena";
|
||||||
|
features: ScrapeEngineFeature[] = ["content", "rating", "tags", "source"];
|
||||||
|
notes = [];
|
||||||
|
supportedHosts = ["derpibooru.org", "trixiebooru.org", "ponybooru.org", "furbooru.org", "ponerpics.org", "manebooru.art", "twibooru.org"];
|
||||||
|
|
||||||
|
scrapeDocument(document: Document): ScrapeResult {
|
||||||
|
const result = new ScrapeResult(this.name);
|
||||||
|
const post = new ScrapedPost();
|
||||||
|
post.pageUrl = document.location.href;
|
||||||
|
|
||||||
|
// Set image url
|
||||||
|
const downloadEl = document.querySelector("div[id^='image_meta_'] > div:nth-child(4) > a:nth-child(4)") as HTMLAnchorElement;
|
||||||
|
post.contentUrl = downloadEl?.href;
|
||||||
|
|
||||||
|
// Set content type
|
||||||
|
if (document.querySelector("video#image-display") != null) {
|
||||||
|
post.contentType = "video";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set default rating (Will be determined later)
|
||||||
|
post.rating = "unsafe";
|
||||||
|
|
||||||
|
// Set tags
|
||||||
|
const tagEls = Array.from(document.querySelectorAll("a.tag__name")).map((x) => x as HTMLLIElement);
|
||||||
|
// Workaround for forked builds of philomena
|
||||||
|
if (tagEls.length === 0) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const el of tagEls) {
|
||||||
|
const tagName = (el.innerText);
|
||||||
|
let category: TagCategory | undefined;
|
||||||
|
|
||||||
|
// Set rating (philomena uses tags for this)
|
||||||
|
switch (tagName) {
|
||||||
|
case "safe":
|
||||||
|
post.rating = "safe";
|
||||||
|
break;
|
||||||
|
case "suggestive":
|
||||||
|
post.rating = "sketchy";
|
||||||
|
break;
|
||||||
|
case "questionable":
|
||||||
|
post.rating = "sketchy";
|
||||||
|
break;
|
||||||
|
case "explicit":
|
||||||
|
post.rating = "unsafe";
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tagName) {
|
||||||
|
const tag = new ScrapedTag(tagName, category);
|
||||||
|
post.tags.push(tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set source
|
||||||
|
const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement);
|
||||||
|
post.sources = sourceEls.map((x) => x.href);
|
||||||
|
|
||||||
|
result.tryAddPost(post);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -7,6 +7,7 @@ export { default as FurAffinity } from "./FurAffinity.js";
|
||||||
export { default as Gelbooru } from "./Gelbooru.js";
|
export { default as Gelbooru } from "./Gelbooru.js";
|
||||||
export { default as Inkbunny } from "./Inkbunny.js";
|
export { default as Inkbunny } from "./Inkbunny.js";
|
||||||
export { default as Moebooru } from "./Moebooru.js";
|
export { default as Moebooru } from "./Moebooru.js";
|
||||||
|
export { default as Philomena } from "./Philomena.js";
|
||||||
export { default as Pixiv } from "./Pixiv.js";
|
export { default as Pixiv } from "./Pixiv.js";
|
||||||
export { default as Reddit } from "./Reddit.js";
|
export { default as Reddit } from "./Reddit.js";
|
||||||
export { default as Rule34us } from "./Rule34us.js";
|
export { default as Rule34us } from "./Rule34us.js";
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue