Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix twitter engine for new site design

pull/16/head
Ben Klein 2021-10-03 02:12:44 -04:00 committed by neobooru
parent 352ba505e1
commit d1a6be0079
1 changed files with 25 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
src/engines/Twitter.ts
View File

@ -1,48 +1,46 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { guessContentType } from "../Utility"; import { guessContentType } from "../Utility";
export function getOriginalImageUrl(imageUrl: string) { export function getOriginalImageUrl(image: HTMLImageElement) {
const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url // if the link includes a resolution, change it to orig:
return x + ":orig"; // Add :orig to url (to get original image) // https://pbs.twimg.com/media/stuff?format=jpg&name=900x900
// -> https://pbs.twimg.com/media/stuff?format=jpg&name=orig
let img_url = new URL((image as HTMLImageElement).src);
if (img_url.searchParams.has('name') && img_url.searchParams.get('name') != 'orig') {
img_url.searchParams.set('name', 'orig');
}
return img_url;
} }
export default class Twitter implements ScrapeEngine { export default class Twitter implements ScrapeEngine {
name = "twitter"; name = "twitter";
canImport(url: Location): boolean { canImport(url: Location): boolean {
return url.host == "twitter.com"; return url.host == "twitter.com" || url.host == "mobile.twitter.com";
} }
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url // first look for a focused image on desktop:
// When clicked on post (not fullscreen, with post text etc) const center_img = document.querySelector("div[role=dialog] div[role=dialog] div[aria-label=Image] img");
const overlayElement = document.querySelector("#permalink-overlay-dialog"); if (center_img) {
if (overlayElement) { let post = new ScrapedPost();
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img"); post.pageUrl = document.location.href;
if (cardImageElement) { post.contentUrl = getOriginalImageUrl(center_img as HTMLImageElement).href;
post.contentUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src); result.tryAddPost(post);
}
} }
// When clicked on image (fullscreen) // look for images within a tweet:
const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image"); const imgs = [...document.querySelectorAll("article a[role='link'] div[data-testid='tweetPhoto'] img")];
if (mediaImageElements.length > 0) {
post.contentUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src); for (const img of imgs) {
let post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = getOriginalImageUrl(img as HTMLImageElement).href;
result.tryAddPost(post);
} }
if (post.contentUrl == undefined) {
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
result.tryAddPost(post);
return result; return result;
} }
} }