| 1 | const verbose = false |
| 2 | const global_lock = [] |
| 3 | const google_api_key = '' |
| 4 | const google_api_cs = '' |
| 5 | const grid_url = '' |
| 6 | const proxy = '' |
| 7 | let tecert_file = '' |
| 8 | |
| 9 | const detection_level = { |
| 10 | extreme: { |
| 11 | fast: 'normal', |
| 12 | slow: 'normal,advanced,ocr', |
| 13 | detections: 'true', |
| 14 | count: 1, |
| 15 | found: 2 |
| 16 | }, |
| 17 | high: { |
| 18 | fast: 'normal', |
| 19 | slow: 'normal,advanced,ocr', |
| 20 | detections: 'true,false', |
| 21 | count: 2, |
| 22 | found: 1 |
| 23 | }, |
| 24 | current: 'high' |
| 25 | } |
| 26 | |
| 27 | const profile_template = { |
| 28 | found: 0, |
| 29 | username: '', |
| 30 | image: '', |
| 31 | link: '', |
| 32 | rate: '', |
| 33 | status: '', |
| 34 | title: '', |
| 35 | language: '', |
| 36 | country: '', |
| 37 | rank: '', |
| 38 | text: '', |
| 39 | type: '', |
| 40 | metadata: '', |
| 41 | extracted: '', |
| 42 | good: '', |
| 43 | method: '' |
| 44 | } |
| 45 | |
| 46 | const detected_websites = { |
| 47 | normal: 0, |
| 48 | advanced: 0, |
| 49 | ocr: 0, |
| 50 | true: 0, |
| 51 | false: 0, |
| 52 | count: 0 |
| 53 | } |
| 54 | |
| 55 | const header_options = { |
| 56 | headers: { |
| 57 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0' |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | |
| 62 | import https from 'follow-redirects' |
| 63 | import fs from 'fs' |
| 64 | import url from 'url' |
| 65 | import {franc} from 'franc' |
| 66 | import langs from 'langs' |
| 67 | import * as cheerio from 'cheerio' |
| 68 | import path from 'path' |
| 69 | import slash from 'slash' |
| 70 | import colors from 'colors/safe.js' |
| 71 | import {QBIxora} from 'ixora' |
| 72 | import {fileURLToPath} from 'url'; |
| 73 | |
| 74 | const __dirname = path.dirname(fileURLToPath(import.meta.url)); |
| 75 | const sites_json_path = slash(path.join(__dirname, '..', 'data', 'sites.json')) |
| 76 | const names_json_path = slash(path.join(__dirname, '..', 'data', 'names.json')) |
| 77 | const dict_json_path = slash(path.join(__dirname, '..', 'data', 'dict.json')) |
| 78 | const countries_json_path = slash(path.join(__dirname, '..', 'data', 'names.json')) |
| 79 | const public_graph_path = slash(path.join(__dirname, '..', 'public', 'graph.html')) |
| 80 | |
| 81 | let temp_ixora = new QBIxora('Social-Analyzer', false) |
| 82 | temp_ixora.save_base_html(public_graph_path) |
| 83 | temp_ixora = null |
| 84 | |
| 85 | const websites_entries = JSON.parse(fs.readFileSync(sites_json_path)).websites_entries |
| 86 | const shared_detections = JSON.parse(fs.readFileSync(sites_json_path)).shared_detections |
| 87 | const parsed_names_origins = JSON.parse(fs.readFileSync(names_json_path)) |
| 88 | const parsed_json = JSON.parse(fs.readFileSync(dict_json_path)) |
| 89 | const parsed_countries = JSON.parse(fs.readFileSync(names_json_path)) |
| 90 | |
| 91 | let logs_queue = Promise.resolve() |
| 92 | |
| 93 | const strings_pages = new RegExp('captcha-info|Please enable cookies|Completing the CAPTCHA', 'i') |
| 94 | const strings_titles = new RegExp('not found|blocked|attention required|cloudflare', 'i') |
| 95 | const top_websites = new RegExp('^top([0-9]+)$', 'i') |
| 96 | |
| 97 | function get_log_file (uuid) { |
| 98 | const _uuid = uuid.replace(/[^a-zA-Z0-9\-]+/g, '') |
| 99 | const _string = slash(path.join('logs', _uuid + '_log.txt')) |
| 100 | return _string |
| 101 | } |
| 102 | |
| 103 | function log_to_file_queue (uuid, msg, table = false, argv = undefined) { |
| 104 | logs_queue = logs_queue.then(function () { |
| 105 | return new Promise(function (resolve) { |
| 106 | const temp_log_file = slash(path.join('logs', uuid + '_log.txt')) |
| 107 | fs.appendFile(temp_log_file, msg + '\n', function (err, data) { |
| 108 | if (table) { |
| 109 | msg.forEach((item, index) => { |
| 110 | if (index === 0) { |
| 111 | console.log('-----------------------') |
| 112 | } |
| 113 | for (const [key, value] of Object.entries(item)) { |
| 114 | if (key === 'extracted' || key === 'metadata') { |
| 115 | if ((key === 'extracted' && argv.extract) || (key === 'metadata' && argv.metadata)) { |
| 116 | if (value !== 'unavailable') { |
| 117 | try { |
| 118 | value.forEach((metadata_item, i) => { |
| 119 | let temp_string_meta = key + ' ' + i |
| 120 | temp_string_meta = temp_string_meta.padEnd(13) |
| 121 | temp_string_meta = colors.blue(temp_string_meta) + ': ' |
| 122 | for (const [metadata_key, metadata_value] of Object.entries(metadata_item)) { |
| 123 | if (metadata_value.length > 80 && argv.trim) { |
| 124 | temp_string_meta += colors.blue(metadata_key) + ' : ' + colors.yellow(metadata_value.substring(0, 80).replace(/\r?\n|\r/g, '') + '..') + ' ' |
| 125 | } else { |
| 126 | temp_string_meta += colors.blue(metadata_key) + ' : ' + colors.yellow(metadata_value.replace(/\r?\n|\r/g, '')) + ' ' |
| 127 | } |
| 128 | }; |
| 129 | console.log(temp_string_meta) |
| 130 | }) |
| 131 | } catch (err) { |
| 132 | |
| 133 | } |
| 134 | } else { |
| 135 | console.log(colors.blue(key.padEnd(12)) + ' : ' + colors.yellow(value)) |
| 136 | } |
| 137 | } |
| 138 | } else { |
| 139 | console.log(colors.blue(key.padEnd(12)) + ' : ' + colors.yellow(value)) |
| 140 | } |
| 141 | } |
| 142 | console.log('-----------------------') |
| 143 | }) |
| 144 | } else { |
| 145 | console.log(msg) |
| 146 | } |
| 147 | resolve() |
| 148 | }) |
| 149 | }) |
| 150 | }) |
| 151 | } |
| 152 | |
| 153 | function get_language_by_parsing (body) { |
| 154 | let language = 'unavailable' |
| 155 | try { |
| 156 | const $ = cheerio.load(body) |
| 157 | const code = $('html').attr('lang') |
| 158 | if (code !== '') { |
| 159 | if (langs.where('1', code) !== 'undefined' && langs.where('1', code)) { |
| 160 | language = langs.where('1', code).name |
| 161 | } |
| 162 | } |
| 163 | } catch (err) { |
| 164 | verbose && console.log(err) |
| 165 | } |
| 166 | return language |
| 167 | } |
| 168 | |
| 169 | function get_language_by_guessing (text) { |
| 170 | let language = 'unavailable' |
| 171 | try { |
| 172 | if (text !== 'unavailable' && text !== '') { |
| 173 | const code = franc(text) |
| 174 | if (code !== 'und') { |
| 175 | if (langs.where('3', code) !== 'undefined' && langs.where('3', code)) { |
| 176 | language = langs.where('3', code).name + ' (Maybe)' |
| 177 | } |
| 178 | } |
| 179 | } |
| 180 | } catch (err) { |
| 181 | verbose && console.log(err) |
| 182 | } |
| 183 | |
| 184 | return language |
| 185 | } |
| 186 | |
| 187 | function get_site_from_url (_url) { |
| 188 | const temp = url.parse(_url.replace('{username}', 'nothinghere')).hostname |
| 189 | return temp.replace('nothinghere.', '') |
| 190 | } |
| 191 | |
| 192 | async function get_url_wrapper_json (url, time = 2) { |
| 193 | try { |
| 194 | const http_promise = new Promise((resolve, reject) => { |
| 195 | const request = https.https.get(url, header_options, function (res) { |
| 196 | let body = '' |
| 197 | res.on('data', function (chunk) { |
| 198 | body += chunk |
| 199 | }) |
| 200 | res.on('end', function () { |
| 201 | resolve({ |
| 202 | data: JSON.parse(body.toString()) |
| 203 | }) |
| 204 | }) |
| 205 | }) |
| 206 | const timeout = (time !== 0) ? time * 1000 : 5000 |
| 207 | request.setTimeout(timeout, function() { |
| 208 | reject({ |
| 209 | data: '' |
| 210 | }) |
| 211 | }); |
| 212 | request.on('error', function (e) { |
| 213 | reject({ |
| 214 | data: '' |
| 215 | }) |
| 216 | }) |
| 217 | request.on('socket', function (socket) { |
| 218 | const timeout = (time !== 0) ? time * 1000 : 5000 |
| 219 | socket.setTimeout(timeout, function () { |
| 220 | request.abort() |
| 221 | }) |
| 222 | }) |
| 223 | }) |
| 224 | const response_body = await http_promise |
| 225 | return response_body |
| 226 | } catch (err) { |
| 227 | verbose && console.log(err) |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | async function get_url_wrapper_text (url, time = 2) { |
| 232 | const response_body = 'error-get-url' |
| 233 | const ret = 500 |
| 234 | try { |
| 235 | const http_promise = new Promise((resolve, reject) => { |
| 236 | const request = https.https.get(url, header_options, function (res) { |
| 237 | let body = '' |
| 238 | res.on('data', function (chunk) { |
| 239 | body += chunk |
| 240 | }) |
| 241 | res.on('end', function () { |
| 242 | resolve([res.statusCode,body]) |
| 243 | }) |
| 244 | }) |
| 245 | const timeout = (time !== 0) ? time * 1000 : 5000 |
| 246 | request.setTimeout(timeout, function() { |
| 247 | reject({ |
| 248 | data: '' |
| 249 | }) |
| 250 | }); |
| 251 | request.on('error', function (e) { |
| 252 | reject({ |
| 253 | data: '' |
| 254 | }) |
| 255 | }) |
| 256 | request.on('socket', function (socket) { |
| 257 | const timeout = (time !== 0) ? time * 1000 : 5000 |
| 258 | socket.setTimeout(timeout, function () { |
| 259 | request.abort() |
| 260 | }) |
| 261 | }) |
| 262 | }) |
| 263 | const [ret, response_body] = await http_promise |
| 264 | return [ret, response_body] |
| 265 | } catch (err) { |
| 266 | verbose && console.log(err) |
| 267 | return [ret, response_body] |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | function compare_objects (object1, object2, key) { |
| 272 | try { |
| 273 | if (object1[key] === '') { |
| 274 | object1[key] = '%0.0' |
| 275 | } |
| 276 | if (object2[key] === '') { |
| 277 | object2[key] = '%0.0' |
| 278 | } |
| 279 | if (parseInt(object1[key].replace('%', '')) > parseInt(object2[key].replace('%', ''))) { |
| 280 | return -1 |
| 281 | } else if (parseInt(object1[key].replace('%', '')) < parseInt(object2[key].replace('%', ''))) { |
| 282 | return 1 |
| 283 | } else { |
| 284 | return 0 |
| 285 | } |
| 286 | } catch (err) { |
| 287 | return 0 |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | function find_country (code) { |
| 292 | let ctr = '' |
| 293 | try { |
| 294 | if (code.toUpperCase() in parsed_countries) { |
| 295 | ctr = parsed_countries[code.toUpperCase()] |
| 296 | } |
| 297 | } catch (error) { |
| 298 | |
| 299 | } |
| 300 | return ctr |
| 301 | } |
| 302 | |
| 303 | async function setup_tecert () { |
| 304 | if (!fs.existsSync('eng.traineddata')) { |
| 305 | const file = fs.createWriteStream('eng.traineddata') |
| 306 | const http_promise = new Promise((resolve, reject) => { |
| 307 | const request = https.https.get('https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata', function (response) { |
| 308 | response.pipe(file) |
| 309 | resolve(1) |
| 310 | request.setTimeout(12000, function () { |
| 311 | request.abort() |
| 312 | }) |
| 313 | file.on('finish', function () { |
| 314 | file.close() |
| 315 | }) |
| 316 | resolve(0) |
| 317 | }) |
| 318 | }) |
| 319 | |
| 320 | const get_eng = await http_promise |
| 321 | if (get_eng === 1) { |
| 322 | if (fs.existsSync('eng.traineddata')) { |
| 323 | tecert_file = path.resolve(__dirname, 'eng.traineddata') |
| 324 | } |
| 325 | } |
| 326 | } else { |
| 327 | if (tecert_file === '') { |
| 328 | if (fs.existsSync('eng.traineddata')) { |
| 329 | tecert_file = path.resolve(__dirname, 'eng.traineddata') |
| 330 | } |
| 331 | } |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | export default { |
| 336 | strings_pages, |
| 337 | strings_titles, |
| 338 | top_websites, |
| 339 | tecert_file, |
| 340 | setup_tecert, |
| 341 | compare_objects, |
| 342 | get_log_file, |
| 343 | find_country, |
| 344 | profile_template, |
| 345 | detection_level, |
| 346 | detected_websites, |
| 347 | shared_detections, |
| 348 | get_language_by_parsing, |
| 349 | get_language_by_guessing, |
| 350 | websites_entries, |
| 351 | parsed_names_origins, |
| 352 | parsed_json, |
| 353 | verbose, |
| 354 | global_lock, |
| 355 | google_api_key, |
| 356 | google_api_cs, |
| 357 | grid_url, |
| 358 | header_options, |
| 359 | proxy, |
| 360 | get_site_from_url, |
| 361 | log_to_file_queue, |
| 362 | get_url_wrapper_text, |
| 363 | get_url_wrapper_json |
| 364 | } |
| 365 |