| 1 | #!/usr/bin/env python |
| 2 | |
| 3 | """ |
| 4 | // ------------------------------------------------------------- |
| 5 | // author Giga |
| 6 | // project qeeqbox/social-analyzer |
| 7 | // email gigaqeeq@gmail.com |
| 8 | // description app.py (CLI) |
| 9 | // licensee AGPL-3.0 |
| 10 | // ------------------------------------------------------------- |
| 11 | // contributors list qeeqbox/social-analyzer/graphs/contributors |
| 12 | // ------------------------------------------------------------- |
| 13 | """ |
| 14 | |
| 15 | from logging import getLogger, DEBUG, Formatter, Handler, addLevelName, NullHandler |
| 16 | from logging.handlers import RotatingFileHandler |
| 17 | from sys import platform, version_info |
| 18 | from sys import argv as sargv |
| 19 | from os import path, system, makedirs |
| 20 | from time import time, sleep |
| 21 | from argparse import ArgumentParser, SUPPRESS, Namespace |
| 22 | from json import load, dumps, loads |
| 23 | from uuid import uuid4 |
| 24 | from collections.abc import Mapping |
| 25 | from functools import wraps |
| 26 | from re import sub as resub |
| 27 | from re import findall, IGNORECASE |
| 28 | from re import compile as recompile |
| 29 | from re import search as research |
| 30 | from contextlib import suppress |
| 31 | from concurrent.futures import ThreadPoolExecutor, as_completed |
| 32 | from random import randint |
| 33 | from tempfile import mkdtemp |
| 34 | from urllib.parse import unquote, urlparse |
| 35 | from urllib3.exceptions import InsecureRequestWarning |
| 36 | from bs4 import BeautifulSoup |
| 37 | from tld import get_fld, get_tld |
| 38 | from requests import get, packages, Session |
| 39 | from termcolor import colored |
| 40 | from langdetect import detect |
| 41 | from warnings import filterwarnings |
| 42 | from galeodes import Galeodes |
| 43 | |
| 44 | filterwarnings('ignore', category=RuntimeWarning, module='runpy') |
| 45 | packages.urllib3.disable_warnings(category=InsecureRequestWarning) |
| 46 | filterwarnings("ignore", category=UserWarning, module='bs4') |
| 47 | |
| 48 | |
| 49 | class SocialAnalyzer(): |
| 50 | def __init__(self, silent=False): |
| 51 | self.websites_entries = [] |
| 52 | self.shared_detections = [] |
| 53 | self.generic_detection = [] |
| 54 | self.log = getLogger("social-analyzer") |
| 55 | self.sites_path = path.join(path.dirname(__file__), "data", "sites.json") |
| 56 | self.languages_path = path.join(path.dirname(__file__), "data", "languages.json") |
| 57 | self.strings_pages = recompile('captcha-info|Please enable cookies|Completing the CAPTCHA', IGNORECASE) |
| 58 | self.strings_titles = recompile('not found|blocked|attention required|cloudflare', IGNORECASE) |
| 59 | self.strings_meta = recompile(r'regionsAllowed|width|height|color|rgba\(|charset|viewport|refresh|equiv|robots', IGNORECASE) |
| 60 | self.top_pattern = recompile('^top([0-9]+)$', IGNORECASE) |
| 61 | self.languages_json = None |
| 62 | self.sites_dummy = None |
| 63 | self.workers = 15 |
| 64 | self.custom_message = 51 |
| 65 | self.timeout = None |
| 66 | self.waf = True |
| 67 | self.logs_dir = '' |
| 68 | self.ret = False |
| 69 | self.headers = {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0", } |
| 70 | self.silent = silent |
| 71 | self.screenshots = None |
| 72 | self.screenshots_location = None |
| 73 | |
| 74 | def delete_keys(self, in_object, keys): |
| 75 | ''' |
| 76 | delete specific keys from object |
| 77 | ''' |
| 78 | |
| 79 | for key in keys: |
| 80 | with suppress(Exception): |
| 81 | del in_object[key] |
| 82 | return in_object |
| 83 | |
| 84 | def clean_up_item(self, in_object, keys_str): |
| 85 | ''' |
| 86 | delete specific keys from object (user input) |
| 87 | ''' |
| 88 | |
| 89 | with suppress(Exception): |
| 90 | del in_object["image"] |
| 91 | if keys_str == "" or keys_str is None: |
| 92 | with suppress(Exception): |
| 93 | pass |
| 94 | else: |
| 95 | for key in in_object.copy(): |
| 96 | if key not in keys_str: |
| 97 | with suppress(Exception): |
| 98 | del in_object[key] |
| 99 | return in_object |
| 100 | |
| 101 | def get_language_by_guessing(self, text): |
| 102 | ''' |
| 103 | guess language by text, this needs long text |
| 104 | ''' |
| 105 | |
| 106 | with suppress(Exception): |
| 107 | lang = detect(text) |
| 108 | if lang and lang != "": |
| 109 | return self.languages_json[lang] + " (Maybe)" |
| 110 | return "unavailable" |
| 111 | |
| 112 | def get_language_by_parsing(self, source, encoding): |
| 113 | ''' |
| 114 | guess language by parsing the lang tag |
| 115 | ''' |
| 116 | |
| 117 | with suppress(Exception): |
| 118 | lang = BeautifulSoup(source, "html.parser", from_encoding=encoding).find("html", attrs={"lang": True})["lang"] |
| 119 | if lang and lang != "": |
| 120 | return self.languages_json[lang] |
| 121 | return "unavailable" |
| 122 | |
| 123 | def check_errors(self, on_off=None): |
| 124 | ''' |
| 125 | wrapper function for debugging |
| 126 | ''' |
| 127 | |
| 128 | def decorator(func): |
| 129 | @wraps(func) |
| 130 | def wrapper(*args, **kwargs): |
| 131 | if on_off: |
| 132 | try: |
| 133 | return func(*args, **kwargs) |
| 134 | except Exception as err: |
| 135 | pass |
| 136 | # if not self.silent: self.log.info(e) |
| 137 | else: |
| 138 | return func(*args, **kwargs) |
| 139 | return wrapper |
| 140 | return decorator |
| 141 | |
| 142 | def setup_logger(self, uuid=None, file=False, argv=None): |
| 143 | ''' |
| 144 | setup a logger for logs in the temp folder |
| 145 | ''' |
| 146 | |
| 147 | class CustomHandler(Handler): |
| 148 | ''' |
| 149 | custom stream handler |
| 150 | ''' |
| 151 | |
| 152 | def __init__(self, argv=None, sa_object=None): |
| 153 | ''' |
| 154 | int, user choices needed |
| 155 | ''' |
| 156 | |
| 157 | Handler.__init__(self) |
| 158 | self.argv = argv |
| 159 | self.sa_object = sa_object |
| 160 | |
| 161 | def emit(self, record): |
| 162 | ''' |
| 163 | emit, based on user choices |
| 164 | ''' |
| 165 | |
| 166 | if self.argv.output != "json" and self.sa_object.silent == False: |
| 167 | if isinstance(record.msg, Mapping): |
| 168 | if "custom" in record.msg: |
| 169 | for item in record.msg["custom"]: |
| 170 | with suppress(Exception): |
| 171 | if item == record.msg["custom"][0]: |
| 172 | print("-----------------------") |
| 173 | for key, value in item.items(): |
| 174 | if key == "metadata" or key == "extracted": |
| 175 | if (self.argv.metadata and key == "metadata") or (self.argv.extract and key == "extracted"): |
| 176 | with suppress(Exception): |
| 177 | for idx, _item in enumerate(value): |
| 178 | empty_string = key + " " + str(idx) |
| 179 | empty_string = colored(empty_string.ljust(13, ' '), 'blue') + ": " |
| 180 | for _item_key, _item_value in _item.items(): |
| 181 | if self.argv.trim and _item_key == "content" and len(_item_value) > 50: |
| 182 | empty_string += "{} : {} ".format(colored(_item_key, 'blue'), colored(_item_value[:50] + "..", 'yellow')) |
| 183 | else: |
| 184 | empty_string += "{} : {} ".format(colored(_item_key, 'blue'), colored(_item_value, 'yellow')) |
| 185 | print("{}".format(empty_string)) |
| 186 | else: |
| 187 | print(colored(key.ljust(13, ' '), 'blue'), colored(value, 'yellow'), sep=": ") |
| 188 | print("-----------------------") |
| 189 | else: |
| 190 | print(record.msg) |
| 191 | |
| 192 | temp_folder = '' |
| 193 | if argv.logs: |
| 194 | if self.logs_dir != '': |
| 195 | temp_folder = self.logs_dir |
| 196 | else: |
| 197 | temp_folder = mkdtemp() |
| 198 | |
| 199 | if file and uuid: |
| 200 | if argv.screenshots: |
| 201 | self.screenshots = True |
| 202 | makedirs(path.join(temp_folder, uuid), exist_ok=True) |
| 203 | self.screenshots_location = path.join(temp_folder, uuid) |
| 204 | fh = RotatingFileHandler(path.join(temp_folder, uuid, 'logs')) |
| 205 | fh.setFormatter(Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) |
| 206 | self.log.addHandler(fh) |
| 207 | |
| 208 | self.log.setLevel(DEBUG) |
| 209 | |
| 210 | if argv.silent: |
| 211 | self.log.addHandler(NullHandler()) |
| 212 | else: |
| 213 | self.log.addHandler(CustomHandler(argv, sa_object=self)) |
| 214 | |
| 215 | if argv.logs and argv.output != "json": |
| 216 | if not self.silent: |
| 217 | self.log.info('[init] Temporary Logs Directory {}'.format(temp_folder)) |
| 218 | |
| 219 | def init_detections(self, detections): |
| 220 | ''' |
| 221 | load websites_entries, shared_detections and generic_detection |
| 222 | ''' |
| 223 | |
| 224 | temp_list = [] |
| 225 | with open(self.sites_path, encoding='utf-8') as file: |
| 226 | for item in load(file)[detections]: |
| 227 | item["selected"] = "false" |
| 228 | temp_list.append(item) |
| 229 | return temp_list |
| 230 | |
| 231 | def get_website(self, site): |
| 232 | ''' |
| 233 | extract domain from website |
| 234 | ''' |
| 235 | |
| 236 | temp_value = get_fld(site, fix_protocol=True) |
| 237 | temp_value = temp_value.replace(".{username}", "").replace("{username}.", "") |
| 238 | return temp_value |
| 239 | |
| 240 | def search_and_change(self, site, _dict): |
| 241 | with suppress(Exception): |
| 242 | if site in self.websites_entries: |
| 243 | item = self.websites_entries.index(site) |
| 244 | self.websites_entries[item].update(_dict) |
| 245 | |
| 246 | def top_websites(self, top_number): |
| 247 | with suppress(Exception): |
| 248 | top_websites = research(self.top_pattern, top_number) |
| 249 | if top_websites: |
| 250 | sites = ([d for d in self.websites_entries if d.get('global_rank') != 0]) |
| 251 | sites = sorted(sites, key=lambda x: x['global_rank']) |
| 252 | for site in sites[:int(top_websites.group(1))]: |
| 253 | self.search_and_change(site, {"selected": "true"}) |
| 254 | return True |
| 255 | return False |
| 256 | |
| 257 | def list_all_websites(self): |
| 258 | ''' |
| 259 | list all the available websites' entries |
| 260 | ''' |
| 261 | if len(self.websites_entries) > 0: |
| 262 | for site in self.websites_entries: |
| 263 | temp_value = get_fld(site["url"], fix_protocol=True) |
| 264 | temp_value = temp_value.replace(".{username}", "").replace("{username}.", "") |
| 265 | if not self.silent: |
| 266 | self.log.info(temp_value) |
| 267 | |
| 268 | def fetch_url(self, site, username, options): |
| 269 | ''' |
| 270 | this runs for every website entry |
| 271 | ''' |
| 272 | |
| 273 | if self.timeout: |
| 274 | sleep(self.timeout) |
| 275 | else: |
| 276 | sleep(randint(1, 99) / 100) |
| 277 | |
| 278 | checking_url = None |
| 279 | with suppress(Exception): |
| 280 | checking_url = get_tld(site["url"], as_object=True).parsed_url.netloc |
| 281 | if checking_url is None: |
| 282 | checking_url = get_fld(site["url"]) |
| 283 | checking_url = checking_url.replace(".{username}", "").replace("{username}.", "") |
| 284 | if not self.silent: |
| 285 | self.log.info("[Checking] " + checking_url) |
| 286 | |
| 287 | source = "" |
| 288 | |
| 289 | detection_level = { |
| 290 | "extreme": { |
| 291 | "fast": "normal", |
| 292 | "slow": "normal,advanced,ocr", |
| 293 | "detections": "true", |
| 294 | "count": 1, |
| 295 | "found": 2 |
| 296 | }, |
| 297 | "high": { |
| 298 | "fast": "normal", |
| 299 | "slow": "normal,advanced,ocr", |
| 300 | "detections": "true,false", |
| 301 | "count": 2, |
| 302 | "found": 1 |
| 303 | }, |
| 304 | "current": "high" |
| 305 | } |
| 306 | |
| 307 | with suppress(Exception): |
| 308 | session = Session() |
| 309 | session.headers.update(self.headers) |
| 310 | response = session.get(site["url"].replace("{username}", username), timeout=5, verify=False) |
| 311 | source = response.text |
| 312 | content = response.content |
| 313 | encoding = response.encoding |
| 314 | answer = dict((k.lower(), v.lower()) for k, v in response.headers.items()) |
| 315 | session.close() |
| 316 | temp_profile = {} |
| 317 | temp_detected = {} |
| 318 | detections_count = 0 |
| 319 | |
| 320 | def check_url(url): |
| 321 | ''' |
| 322 | check if url is okay |
| 323 | ''' |
| 324 | |
| 325 | with suppress(Exception): |
| 326 | result = urlparse(url) |
| 327 | if result.scheme == "http" or result.scheme == "https": |
| 328 | return all([result.scheme, result.netloc]) |
| 329 | return False |
| 330 | |
| 331 | def merge_dicts(temp_dict): |
| 332 | ''' |
| 333 | ''' |
| 334 | |
| 335 | result = {} |
| 336 | for item in temp_dict: |
| 337 | for key, value in item.items(): |
| 338 | if key in result: |
| 339 | result[key] += value |
| 340 | else: |
| 341 | result[key] = value |
| 342 | return result |
| 343 | |
| 344 | def detect_logic(detections): |
| 345 | ''' |
| 346 | check for detections in website entry |
| 347 | ''' |
| 348 | |
| 349 | detections_count = 0 |
| 350 | temp_detected = [] |
| 351 | temp_found = "false" |
| 352 | temp_profile = { |
| 353 | "found": 0, |
| 354 | "image": "", |
| 355 | "link": "", |
| 356 | "rate": "", |
| 357 | "status": "", |
| 358 | "title": "", |
| 359 | "language": "", |
| 360 | "country": "", |
| 361 | "rank": "", |
| 362 | "text": "", |
| 363 | "type": "", |
| 364 | "extracted": "", |
| 365 | "metadata": "", |
| 366 | "good": "", |
| 367 | "method": "" |
| 368 | } |
| 369 | |
| 370 | for detection in detections: |
| 371 | temp_found = "false" |
| 372 | if detection["type"] in detection_level[detection_level["current"]]["fast"] and source != "": |
| 373 | detections_count += 1 |
| 374 | if detection["string"].replace("{username}", username).lower() in source.lower(): |
| 375 | temp_found = "true" |
| 376 | if detection["return"] == temp_found: |
| 377 | temp_profile["found"] += 1 |
| 378 | return temp_profile, temp_detected, detections_count |
| 379 | |
| 380 | def detect(): |
| 381 | ''' |
| 382 | main detect logic |
| 383 | ''' |
| 384 | |
| 385 | temp_profile_all = [] |
| 386 | temp_detected_all = [] |
| 387 | detections_count_all = 0 |
| 388 | for detection in site["detections"]: |
| 389 | detections_ = [] |
| 390 | if detection["type"] == "shared": |
| 391 | detections_ = next(item for item in self.shared_detections if item["name"] == detection['name']) |
| 392 | if len(detections_) > 0: |
| 393 | val1, val2, val3 = detect_logic(detections_["detections"]) |
| 394 | temp_profile_all.append(val1) |
| 395 | detections_count_all += val3 |
| 396 | |
| 397 | val1, val2, val3 = detect_logic(site["detections"]) |
| 398 | temp_profile_all.append(val1) |
| 399 | detections_count_all += val3 |
| 400 | return merge_dicts(temp_profile_all), temp_detected_all, detections_count_all |
| 401 | |
| 402 | temp_profile, temp_detected, detections_count = detect() |
| 403 | |
| 404 | if temp_profile["found"] >= detection_level[detection_level["current"]]["found"] and detections_count >= detection_level[detection_level["current"]]["count"]: |
| 405 | temp_profile["good"] = "true" |
| 406 | |
| 407 | soup = None |
| 408 | |
| 409 | with suppress(Exception): |
| 410 | soup = BeautifulSoup(content, "html.parser", from_encoding=encoding) |
| 411 | |
| 412 | with suppress(Exception): |
| 413 | temp_text_arr = [] |
| 414 | temp_text_list = [] |
| 415 | soup = BeautifulSoup(content, "html.parser", from_encoding=encoding) |
| 416 | for item in soup.stripped_strings: |
| 417 | if item not in temp_text_list: |
| 418 | temp_text_list.append(item) |
| 419 | temp_text_arr.append(repr(item).replace("'", "")) |
| 420 | temp_profile["text"] = " ".join(temp_text_arr) |
| 421 | temp_profile["text"] = resub(r"\s\s+", " ", temp_profile["text"]) |
| 422 | with suppress(Exception): |
| 423 | temp_profile["language"] = self.get_language_by_parsing(source, encoding) |
| 424 | if temp_profile["language"] == "unavailable": |
| 425 | temp_profile["language"] = self.get_language_by_guessing(temp_profile["text"]) |
| 426 | with suppress(Exception): |
| 427 | temp_profile["title"] = BeautifulSoup(source, "html.parser", from_encoding=encoding).title.string |
| 428 | temp_profile["title"] = resub(r"\s\s+", " ", temp_profile["title"]) |
| 429 | |
| 430 | with suppress(Exception): |
| 431 | temp_matches = [] |
| 432 | temp_matches_list = [] |
| 433 | if "extract" in site: |
| 434 | for item in site["extract"]: |
| 435 | matches = findall(item["regex"], source) |
| 436 | for match in matches: |
| 437 | if item["type"] == "link": |
| 438 | if check_url(unquote(match)): |
| 439 | parsed = "{}:({})".format(item["type"], unquote(match)) |
| 440 | if parsed not in temp_matches: |
| 441 | temp_matches.append(parsed) |
| 442 | temp_matches_list.append({"name": item["type"], "value": unquote(match)}) |
| 443 | |
| 444 | if len(temp_matches_list) > 0: |
| 445 | temp_profile["extracted"] = temp_matches_list |
| 446 | |
| 447 | temp_profile["text"] = temp_profile["text"].replace("\n", "").replace("\t", "").replace("\r", "").strip() |
| 448 | temp_profile["title"] = temp_profile["title"].replace("\n", "").replace("\t", "").replace("\r", "").strip() |
| 449 | |
| 450 | if self.waf: |
| 451 | with suppress(Exception): |
| 452 | if 'cf-ray' in answer: |
| 453 | temp_profile["text"] = "filtered" |
| 454 | temp_profile["title"] = "filtered" |
| 455 | elif "server" in answer: |
| 456 | if "cloudflare" in answer["server"]: |
| 457 | temp_profile["text"] = "filtered" |
| 458 | temp_profile["title"] = "filtered" |
| 459 | if research(self.strings_pages, temp_profile["text"]): |
| 460 | temp_profile["text"] = "filtered" |
| 461 | temp_profile["title"] = "filtered" |
| 462 | if research(self.strings_titles, temp_profile["title"]): |
| 463 | temp_profile["text"] = "filtered" |
| 464 | temp_profile["title"] = "filtered" |
| 465 | |
| 466 | with suppress(Exception): |
| 467 | if detections_count != 0: |
| 468 | temp_value = round(((temp_profile["found"] / detections_count) * 100), 2) |
| 469 | temp_profile["rate"] = "%" + str(temp_value) |
| 470 | if temp_value >= 100.00: |
| 471 | temp_profile["status"] = "good" |
| 472 | elif temp_value >= 50.00 and temp_value < 100.00: |
| 473 | temp_profile["status"] = "maybe" |
| 474 | else: |
| 475 | temp_profile["status"] = "bad" |
| 476 | |
| 477 | # copied from qeeqbox osint (pypi) project (currently in-progress) |
| 478 | |
| 479 | with suppress(Exception): |
| 480 | if temp_profile["status"] == "good": |
| 481 | temp_meta_list = [] |
| 482 | temp_for_checking = [] |
| 483 | soup = BeautifulSoup(content, "lxml", from_encoding=encoding) |
| 484 | for meta in soup.find_all('meta'): |
| 485 | if meta not in temp_for_checking and not research(self.strings_meta, str(meta)): |
| 486 | temp_for_checking.append(meta) |
| 487 | temp_mata_item = {} |
| 488 | add = True |
| 489 | if meta.has_attr("property"): |
| 490 | temp_mata_item.update({"property": meta["property"]}) |
| 491 | if meta.has_attr("content"): |
| 492 | if meta["content"].replace("\n", "").replace("\t", "").replace("\r", "").strip() != "": |
| 493 | temp_mata_item.update({"content": meta["content"].replace("\n", "").replace("\t", "").replace("\r", "").strip()}) |
| 494 | if meta.has_attr("itemprop"): |
| 495 | temp_mata_item.update({"itemprop": meta["itemprop"]}) |
| 496 | if meta.has_attr("name"): |
| 497 | temp_mata_item.update({"name": meta["name"]}) |
| 498 | |
| 499 | with suppress(Exception): |
| 500 | if "property" in temp_mata_item: |
| 501 | for i, item in enumerate(temp_meta_list.copy()): |
| 502 | if "property" in item: |
| 503 | if temp_mata_item["property"] == item["property"]: |
| 504 | temp_meta_list[i]["content"] += ", " + temp_mata_item["content"] |
| 505 | add = False |
| 506 | elif "name" in temp_mata_item: |
| 507 | for i, item in enumerate(temp_meta_list.copy()): |
| 508 | if "name" in item: |
| 509 | if temp_mata_item["name"] == item["name"]: |
| 510 | temp_meta_list[i]["content"] += ", " + temp_mata_item["content"] |
| 511 | add = False |
| 512 | elif "itemprop" in temp_mata_item: |
| 513 | for i, item in enumerate(temp_meta_list.copy()): |
| 514 | if "itemprop" in item: |
| 515 | if temp_mata_item["itemprop"] == item["itemprop"]: |
| 516 | temp_meta_list[i]["content"] += ", " + temp_mata_item["content"] |
| 517 | add = False |
| 518 | |
| 519 | if len(temp_mata_item) > 0 and add: |
| 520 | temp_meta_list.append(temp_mata_item) |
| 521 | |
| 522 | if len(temp_meta_list) > 0: |
| 523 | temp_profile["metadata"] = temp_meta_list |
| 524 | |
| 525 | temp_profile["link"] = site["url"].replace("{username}", username) |
| 526 | temp_profile["type"] = site["type"] |
| 527 | temp_profile["country"] = site["country"] |
| 528 | temp_profile["rank"] = site["global_rank"] |
| 529 | |
| 530 | if temp_profile["rank"] == 0: |
| 531 | temp_profile["rank"] = "unavailable" |
| 532 | |
| 533 | for _item in ["title", "language", "text", "type", "metadata", "extracted", "country"]: |
| 534 | with suppress(Exception): |
| 535 | if temp_profile[_item] == "": |
| 536 | temp_profile[_item] = "unavailable" |
| 537 | |
| 538 | if "FindUserProfilesFast" in options and "GetUserProfilesFast" not in options: |
| 539 | temp_profile["method"] = "find" |
| 540 | elif "GetUserProfilesFast" in options and "FindUserProfilesFast" not in options: |
| 541 | temp_profile["method"] = "get" |
| 542 | elif "FindUserProfilesFast" in options and "GetUserProfilesFast" in options: |
| 543 | temp_profile["method"] = "all" |
| 544 | |
| 545 | copy_temp_profile = temp_profile.copy() |
| 546 | return 1, site["url"], copy_temp_profile |
| 547 | return None, site["url"], [] |
| 548 | |
| 549 | def find_username_normal(self, req): |
| 550 | ''' |
| 551 | main find usernames logic using ThreadPoolExecutor |
| 552 | ''' |
| 553 | |
| 554 | resutls = [] |
| 555 | |
| 556 | for i in range(3): |
| 557 | self.websites_entries[:] = [d for d in self.websites_entries if d.get('selected') == "true"] |
| 558 | if len(self.websites_entries) > 0: |
| 559 | if len(req["body"]["string"].split(',')) > 1: |
| 560 | if not self.silent: |
| 561 | self.log.info("[Info] usernames: {}".format(", ".join(req["body"]["string"].split(',')))) |
| 562 | else: |
| 563 | if not self.silent: |
| 564 | self.log.info("[Info] username: {}".format(req["body"]["string"])) |
| 565 | with ThreadPoolExecutor(max_workers=self.workers) as executor: |
| 566 | future_fetch_url = [] |
| 567 | for site in self.websites_entries: |
| 568 | for username in req["body"]["string"].split(','): |
| 569 | future_fetch_url.append(executor.submit(self.fetch_url, site, username, req["body"]["options"])) |
| 570 | for future in as_completed(future_fetch_url): |
| 571 | with suppress(Exception): |
| 572 | good, site, data = future.result() |
| 573 | if good: |
| 574 | self.websites_entries[:] = [d for d in self.websites_entries if d.get('url') != site] |
| 575 | resutls.append(data) |
| 576 | else: |
| 577 | if not self.silent: |
| 578 | self.log.info("[Waiting to retry] " + self.get_website(site)) |
| 579 | |
| 580 | self.websites_entries[:] = [d for d in self.websites_entries if d.get('selected') == "true"] |
| 581 | if len(self.websites_entries) > 0: |
| 582 | for site in self.websites_entries: |
| 583 | temp_profile = {"link": "", |
| 584 | "method": "failed"} |
| 585 | temp_profile["link"] = site["url"].replace("{username}", req["body"]["string"]) |
| 586 | resutls.append(temp_profile) |
| 587 | return resutls |
| 588 | |
| 589 | def check_user_cli(self, argv): |
| 590 | ''' |
| 591 | main cli logic |
| 592 | ''' |
| 593 | |
| 594 | temp_detected = {"detected": [], "unknown": [], "failed": []} |
| 595 | temp_options = "GetUserProfilesFast,FindUserProfilesFast" |
| 596 | if argv.method != "": |
| 597 | if argv.method == "find": |
| 598 | temp_options = "FindUserProfilesFast" |
| 599 | if argv.method == "get": |
| 600 | temp_options = "GetUserProfilesFast" |
| 601 | |
| 602 | req = {"body": {"uuid": str(uuid4()), "string": argv.username, "options": temp_options}} |
| 603 | self.setup_logger(uuid=req["body"]["uuid"], file=True, argv=argv) |
| 604 | self.init_logic() |
| 605 | |
| 606 | if argv.cli: |
| 607 | if not self.silent: |
| 608 | self.log.info("[Warning] --cli is not needed and will be removed later on") |
| 609 | |
| 610 | for site in self.websites_entries: |
| 611 | site["selected"] = "false" |
| 612 | |
| 613 | if argv.websites == "all": |
| 614 | list_of_countries = [] |
| 615 | if argv.countries != "all": |
| 616 | list_of_countries = argv.countries.split(" ") |
| 617 | for site in self.websites_entries: |
| 618 | if site["country"] != "" and site["country"].lower() in list_of_countries: |
| 619 | site["selected"] = "true" |
| 620 | else: |
| 621 | site["selected"] = "false" |
| 622 | else: |
| 623 | for site in self.websites_entries: |
| 624 | site["selected"] = "true" |
| 625 | |
| 626 | if argv.type != "all": |
| 627 | sites = ([d for d in self.websites_entries if d.get('selected') == "true"]) |
| 628 | if "adult" in argv.type.lower(): |
| 629 | for site in sites: |
| 630 | if "adult" in site["type"].lower(): |
| 631 | self.search_and_change(site, {"selected": "pendding"}) |
| 632 | for site in self.websites_entries: |
| 633 | if site["selected"] == "pendding": |
| 634 | site["selected"] = "true" |
| 635 | else: |
| 636 | site["selected"] = "false" |
| 637 | |
| 638 | if int(argv.top) != 0: |
| 639 | sites = ([d for d in self.websites_entries if d.get('selected') == "true"]) |
| 640 | sites = ([d for d in sites if d.get('global_rank') != 0]) |
| 641 | sites = sorted(sites, key=lambda x: x['global_rank']) |
| 642 | for site in sites[:int(argv.top)]: |
| 643 | self.search_and_change(site, {"selected": "pendding"}) |
| 644 | for site in self.websites_entries: |
| 645 | if site["selected"] == "pendding": |
| 646 | site["selected"] = "true" |
| 647 | else: |
| 648 | site["selected"] = "false" |
| 649 | else: |
| 650 | for site in self.websites_entries: |
| 651 | for temp in argv.websites.split(" "): |
| 652 | if temp in site["url"]: |
| 653 | site["selected"] = "true" |
| 654 | |
| 655 | true_websites = 0 |
| 656 | for site in self.websites_entries: |
| 657 | if site["selected"] == "true": |
| 658 | true_websites += 1 |
| 659 | |
| 660 | if not self.silent: |
| 661 | self.log.info("[Init] Selected websites: {}".format(true_websites)) |
| 662 | resutls = self.find_username_normal(req) |
| 663 | |
| 664 | if argv.simplify: |
| 665 | argv.filter = "good" |
| 666 | |
| 667 | for item in resutls: |
| 668 | if item is not None: |
| 669 | if item["method"] == "all": |
| 670 | if item["good"] == "true": |
| 671 | item = self.delete_keys(item, ["method", "good"]) |
| 672 | item = self.clean_up_item(item, argv.options) |
| 673 | temp_detected["detected"].append(item) |
| 674 | else: |
| 675 | item = self.delete_keys(item, ["found", "rate", "status", "method", "good", "text", "extracted", "metadata"]) |
| 676 | item = self.clean_up_item(item, argv.options) |
| 677 | temp_detected["unknown"].append(item) |
| 678 | elif item["method"] == "find": |
| 679 | if item["good"] == "true": |
| 680 | item = self.delete_keys(item, ["method", "good"]) |
| 681 | item = self.clean_up_item(item, argv.options) |
| 682 | temp_detected["detected"].append(item) |
| 683 | elif item["method"] == "get": |
| 684 | item = self.delete_keys(item, ["found", "rate", "status", "method", "good", "text", "extracted", "metadata"]) |
| 685 | item = self.clean_up_item(item, argv.options) |
| 686 | temp_detected["unknown"].append(item) |
| 687 | else: |
| 688 | item = self.delete_keys(item, ["found", "rate", "status", "method", "good", "text", "title", "language", "rate", "extracted", "metadata"]) |
| 689 | item = self.clean_up_item(item, argv.options) |
| 690 | temp_detected["failed"].append(item) |
| 691 | |
| 692 | with suppress(Exception): |
| 693 | if len(temp_detected["detected"]) == 0: |
| 694 | del temp_detected["detected"] |
| 695 | else: |
| 696 | if "all" in argv.profiles or "detected" in argv.profiles or argv.simplify: |
| 697 | if argv.filter == "all": |
| 698 | pass |
| 699 | else: |
| 700 | if argv.simplify: |
| 701 | temp_list_profiles_simple = [] |
| 702 | for item in temp_detected["detected"]: |
| 703 | if float(item['rate'].strip('%')) == 100.0: |
| 704 | temp_list_profiles_simple.append(item) |
| 705 | temp_detected["detected"].clear() |
| 706 | for item in temp_list_profiles_simple: |
| 707 | item = self.clean_up_item(item, ["link"]) |
| 708 | temp_detected["detected"].append(item) |
| 709 | else: |
| 710 | temp_detected["detected"] = [item for item in temp_detected["detected"] if item['status'] in argv.filter] |
| 711 | if len(temp_detected["detected"]) > 0: |
| 712 | temp_detected["detected"] = sorted(temp_detected["detected"], key=lambda k: float(k['rate'].strip('%')), reverse=True) |
| 713 | else: |
| 714 | del temp_detected["detected"] |
| 715 | else: |
| 716 | del temp_detected["detected"] |
| 717 | |
| 718 | if len(temp_detected["unknown"]) == 0: |
| 719 | del temp_detected["unknown"] |
| 720 | else: |
| 721 | if "all" in argv.profiles or "unknown" in argv.profiles: |
| 722 | pass |
| 723 | else: |
| 724 | del temp_detected["unknown"] |
| 725 | |
| 726 | if len(temp_detected["failed"]) == 0: |
| 727 | del temp_detected["failed"] |
| 728 | else: |
| 729 | if "all" in argv.profiles or "failed" in argv.profiles: |
| 730 | pass |
| 731 | else: |
| 732 | del temp_detected["failed"] |
| 733 | |
| 734 | if argv.output == "pretty" or argv.output == "": |
| 735 | if 'detected' in temp_detected: |
| 736 | if not self.silent: |
| 737 | self.log.info("[Detected] {} Profile[s]".format(len(temp_detected['detected']))) |
| 738 | if 'unknown' in temp_detected: |
| 739 | if not self.silent: |
| 740 | self.log.info("[unknown] {} Profile[s]".format(len(temp_detected['unknown']))) |
| 741 | if 'failed' in temp_detected: |
| 742 | if not self.silent: |
| 743 | self.log.info("[failed] {} Profile[s]".format(len(temp_detected['failed']))) |
| 744 | |
| 745 | if "detected" in temp_detected: |
| 746 | if self.screenshots and self.screenshots_location: |
| 747 | location = None |
| 748 | with suppress(Exception): |
| 749 | if not self.silent: |
| 750 | self.log.info("[Info] Getting screenshots of {} profiles".format(len([item['link'] for item in temp_detected["detected"]]))) |
| 751 | with suppress(Exception): |
| 752 | g = Galeodes(browser="chrome", arguments=['--headless', self.headers['User-Agent']], options=None, implicit_wait=5, verbose=False) |
| 753 | results = g.get_pages(urls=[item['link'] for item in temp_detected["detected"]], screenshots=True, number_of_workers=10, format='jpeg', base64=False) |
| 754 | for item in results: |
| 755 | if item['image'] is not None: |
| 756 | with suppress(Exception): |
| 757 | file_name = resub(r'[^\w\d-]', '_', item['url']) + '.jpeg' |
| 758 | with open(path.join(self.screenshots_location, file_name), 'wb') as f: |
| 759 | f.write(item['image']) |
| 760 | location = self.screenshots_location |
| 761 | if location: |
| 762 | if not self.silent: |
| 763 | self.log.info("[Info] Screenshots location {}".format(location)) |
| 764 | |
| 765 | if argv.simplify: |
| 766 | if 'unknown' in temp_detected: |
| 767 | del temp_detected["unknown"] |
| 768 | if 'failed' in temp_detected: |
| 769 | del temp_detected["failed"] |
| 770 | |
| 771 | if argv.output == "pretty" or argv.output == "": |
| 772 | if 'detected' in temp_detected: |
| 773 | if not self.silent: |
| 774 | self.log.info({"custom": temp_detected['detected']}) |
| 775 | if 'unknown' in temp_detected: |
| 776 | if not self.silent: |
| 777 | self.log.info({"custom": temp_detected['unknown']}) |
| 778 | if 'failed' in temp_detected: |
| 779 | if not self.silent: |
| 780 | self.log.info({"custom": temp_detected['failed']}) |
| 781 | |
| 782 | if argv.output == "json": |
| 783 | if not self.silent: |
| 784 | print(dumps(temp_detected, sort_keys=True, indent=None)) |
| 785 | |
| 786 | return temp_detected |
| 787 | |
| 788 | def load_file(self, name, path_to_check, url_download): |
| 789 | ret = None |
| 790 | try: |
| 791 | if path.exists(path_to_check) == False: |
| 792 | if not self.silent: |
| 793 | self.log.info("[init] Downloading {} from {}".format(name, url_download)) |
| 794 | file = get(url_download, allow_redirects=True) |
| 795 | with open(path_to_check, 'wb') as f: |
| 796 | f.write(file.content) |
| 797 | if path.exists(path_to_check) == True: |
| 798 | if not self.silent: |
| 799 | self.log.info("[init] {} looks good!".format(name)) |
| 800 | with open(path_to_check, encoding="utf-8") as f: |
| 801 | ret = load(f) |
| 802 | except Exception as e: |
| 803 | if not self.silent: |
| 804 | self.log.info("[!] {} Does not exist! cannot be downloaded...".format(name)) |
| 805 | return ret |
| 806 | |
| 807 | def init_logic(self): |
| 808 | if not self.silent: |
| 809 | self.log.info("[init] Detections are updated very often, make sure to get the most up-to-date ones") |
| 810 | if platform == "win32": |
| 811 | system("color") |
| 812 | makedirs(path.join(path.dirname(__file__), "data"), exist_ok=True) |
| 813 | self.languages_json = self.load_file("languages.json", self.languages_path, "https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/data/languages.json") |
| 814 | self.sites_dummy = self.load_file("sites.json", self.sites_path, "https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/data/sites.json") |
| 815 | self.websites_entries = self.init_detections("websites_entries") |
| 816 | self.shared_detections = self.init_detections("shared_detections") |
| 817 | self.generic_detection = self.init_detections("generic_detection") |
| 818 | if self.languages_json is not None and self.sites_dummy is not None: |
| 819 | if not self.silent: |
| 820 | self.log.info("[init] languages.json & sites.json loaded successfully") |
| 821 | else: |
| 822 | if not self.silent: |
| 823 | self.log.info("[init] languages.json & sites.json did not load, exiting..") |
| 824 | exit() |
| 825 | |
| 826 | def run_as_object(self, cli=False, gui=False, logs_dir='', logs=False, extract=False, filter='good', headers={}, list=False, metadata=False, method='all', mode='fast', options='', output='pretty', profiles='detected', type='all', ret=False, silent=False, timeout=0, trim=False, username='', websites='all', countries='all', top='0', screenshots=False, simplify=False): |
| 827 | ret = {} |
| 828 | if logs_dir != '': |
| 829 | self.logs_dir = logs_dir |
| 830 | if headers != {}: |
| 831 | self.headers = headers |
| 832 | |
| 833 | self.timeout = timeout |
| 834 | self.silent = silent |
| 835 | |
| 836 | _l = locals() |
| 837 | del _l['self'] |
| 838 | ARGV = Namespace(**_l) |
| 839 | |
| 840 | if ARGV.list: |
| 841 | self.setup_logger(argv=ARGV) |
| 842 | self.list_all_websites() |
| 843 | self.init_logic() |
| 844 | elif ARGV.mode == "fast": |
| 845 | if ARGV.username != "" and ARGV.websites != "": |
| 846 | ret = self.check_user_cli(ARGV) |
| 847 | return ret |
| 848 | |
| 849 | def run_as_cli(self): |
| 850 | |
| 851 | class _ArgumentParser(ArgumentParser): |
| 852 | def error(self, message): |
| 853 | self.exit(2, 'Error: %s\n' % (message)) |
| 854 | |
| 855 | ret = {} |
| 856 | ARGV = None |
| 857 | ARG_PARSER = _ArgumentParser(description="Qeeqbox/social-analyzer - API and Web App for analyzing & finding a person's profile across 900+ social media websites (Detections are updated regularly)", usage=SUPPRESS) |
| 858 | ARG_PARSER._action_groups.pop() |
| 859 | ARG_PARSER_OPTIONAL = ARG_PARSER.add_argument_group("Arguments") |
| 860 | ARG_PARSER_OPTIONAL.add_argument("--username", help="E.g. johndoe, john_doe or johndoe9999", metavar="", default="") |
| 861 | ARG_PARSER_OPTIONAL.add_argument("--websites", help="A website or websites separated by space E.g. youtube, tiktok or tumblr", metavar="", default="all") |
| 862 | ARG_PARSER_OPTIONAL.add_argument("--mode", help="Analysis mode E.g.fast -> FindUserProfilesFast, slow -> FindUserProfilesSlow or special -> FindUserProfilesSpecial", metavar="", default="fast") |
| 863 | ARG_PARSER_OPTIONAL.add_argument("--output", help="Show the output in the following format: json -> json output for integration or pretty -> prettify the output", metavar="", default="pretty") |
| 864 | ARG_PARSER_OPTIONAL.add_argument("--options", help="Show the following when a profile is found: link, rate, title or text", metavar="", default="") |
| 865 | ARG_PARSER_OPTIONAL.add_argument("--method", help="find -> show detected profiles, get -> show all profiles regardless detected or not, all -> combine find & get", metavar="", default="all") |
| 866 | ARG_PARSER_OPTIONAL.add_argument("--filter", help="Filter detected profiles by good, maybe or bad, you can do combine them with comma (good,bad) or use all", metavar="", default="good") |
| 867 | ARG_PARSER_OPTIONAL.add_argument("--profiles", help="Filter profiles by detected, unknown or failed, you can do combine them with comma (detected,failed) or use all", metavar="", default="detected") |
| 868 | ARG_PARSER_OPTIONAL.add_argument("--countries", help="select websites by country or countries separated by space as: us br ru", metavar="", default="all") |
| 869 | ARG_PARSER_OPTIONAL.add_argument("--type", help="Select websites by type (Adult, Music etc)", metavar="", default="all") |
| 870 | ARG_PARSER_OPTIONAL.add_argument("--top", help="select top websites as 10, 50 etc...[--websites is not needed]", metavar="", default="0") |
| 871 | ARG_PARSER_OPTIONAL.add_argument("--extract", help="Extract profiles, urls & patterns if possible", action="store_true") |
| 872 | ARG_PARSER_OPTIONAL.add_argument("--metadata", help="Extract metadata if possible (pypi QeeqBox OSINT)", action="store_true") |
| 873 | ARG_PARSER_OPTIONAL.add_argument("--trim", help="Trim long strings", action="store_true") |
| 874 | ARG_PARSER_OPTIONAL.add_argument("--gui", help="Reserved for a gui (Not implemented)", action="store_true") |
| 875 | ARG_PARSER_OPTIONAL.add_argument("--cli", help="Reserved for a cli (Not needed)", action="store_true") |
| 876 | ARG_PARSER_OPTIONAL.add_argument("--screenshots", help="Get screenshots from detected profiles (This needs --logs)", action="store_true") |
| 877 | ARG_PARSER_OPTIONAL.add_argument("--simplify", help="Print the detected profiles only (links)", action="store_true") |
| 878 | ARG_PARSER_LIST = ARG_PARSER.add_argument_group("Listing websites & detections") |
| 879 | ARG_PARSER_LIST.add_argument("--list", help="List all available websites", action="store_true") |
| 880 | ARG_PARSER_SETTINGS = ARG_PARSER.add_argument_group("Setting") |
| 881 | ARG_PARSER_SETTINGS.add_argument("--headers", help="Headers as dict", metavar="", default={}, type=loads) |
| 882 | ARG_PARSER_SETTINGS.add_argument("--logs", help="Turn logs on or off", action="store_true") |
| 883 | ARG_PARSER_SETTINGS.add_argument("--logs_dir", help="Change logs directory", metavar="", default="") |
| 884 | ARG_PARSER_SETTINGS.add_argument("--timeout", help="Change timeout between each request", metavar="", type=int, default=0) |
| 885 | ARG_PARSER_SETTINGS.add_argument("--silent", help="Disable output to screen", action="store_true") |
| 886 | |
| 887 | ARGV = ARG_PARSER.parse_args() |
| 888 | if ARGV.logs_dir != '': |
| 889 | self.logs_dir = ARGV.logs_dir |
| 890 | if ARGV.headers != {}: |
| 891 | self.headers = ARGV.headers |
| 892 | |
| 893 | self.timeout = ARGV.timeout |
| 894 | self.silent = ARGV.silent |
| 895 | |
| 896 | if ARGV.list: |
| 897 | self.setup_logger(argv=ARGV) |
| 898 | self.list_all_websites() |
| 899 | self.init_logic() |
| 900 | elif ARGV.mode == "fast": |
| 901 | if ARGV.username != "" and ARGV.websites != "": |
| 902 | ret = self.check_user_cli(ARGV) |
| 903 | return ret |
| 904 | |
| 905 | |
| 906 | def main_logic(): |
| 907 | sa = SocialAnalyzer() |
| 908 | sa.run_as_cli() |
| 909 | |
| 910 | |
| 911 | if __name__ == "__main__": |
| 912 | main_logic() |
| 913 |