| 1 | import helper from './helper.js' |
| 2 | import * as cheerio from 'cheerio' |
| 3 | |
| 4 | |
| 5 | const strings_meta = new RegExp('regionsAllowed|width|height|color|rgba\\(|charset|viewport|refresh|equiv', 'i') |
| 6 | async function extract_metadata (site, source) { |
| 7 | try { |
| 8 | const $ = cheerio.load(source) |
| 9 | const meta = $('meta') |
| 10 | const temp_metadata_list = [] |
| 11 | const temp_metadata_for_checking = [] |
| 12 | Object.keys(meta).forEach(function (key) { |
| 13 | if (meta[key].attribs) { |
| 14 | if (!temp_metadata_for_checking.includes(meta[key].attribs) && !strings_meta.test(JSON.stringify(meta[key].attribs))) { |
| 15 | temp_metadata_for_checking.push(meta[key].attribs) |
| 16 | const temp_dict = {} |
| 17 | let add = true |
| 18 | |
| 19 | if (meta[key].attribs.property) { |
| 20 | temp_dict.property = meta[key].attribs.property |
| 21 | } |
| 22 | if (meta[key].attribs.name) { |
| 23 | temp_dict.name = meta[key].attribs.name |
| 24 | } |
| 25 | if (meta[key].attribs.itemprop) { |
| 26 | temp_dict.itemprop = meta[key].attribs.itemprop |
| 27 | } |
| 28 | if (meta[key].attribs.content) { |
| 29 | if (meta[key].attribs.content.replace('\n', '').replace('\t', '').replace('\r', '').trim() !== '') { |
| 30 | temp_dict.content = meta[key].attribs.content |
| 31 | } |
| 32 | } |
| 33 | |
| 34 | ['property', 'name', 'itemprop'].forEach((item, i) => { |
| 35 | if (temp_dict[item]) { |
| 36 | temp_metadata_list.forEach((_item, i) => { |
| 37 | if (_item[item]) { |
| 38 | if (_item[item] === temp_dict[item]) { |
| 39 | temp_metadata_list[i].content += ', ' + temp_dict.content |
| 40 | add = false |
| 41 | } |
| 42 | } |
| 43 | }) |
| 44 | } |
| 45 | }) |
| 46 | |
| 47 | if (add && Object.keys(temp_dict).length !== 0) { |
| 48 | temp_metadata_list.push(temp_dict) |
| 49 | } |
| 50 | } |
| 51 | } |
| 52 | }) |
| 53 | return temp_metadata_list |
| 54 | } catch (err) { |
| 55 | helper.verbose && console.log(err) |
| 56 | return [] |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | async function extract_patterns (site, source) { |
| 61 | try { |
| 62 | const temp_patterns_list = [] |
| 63 | const temp_patterns_for_checking = [] |
| 64 | if ('extract' in site) { |
| 65 | site.extract.forEach((item, i) => { |
| 66 | const regex_pattern = new RegExp(item.regex, 'g') |
| 67 | let found = null |
| 68 | while (found = regex_pattern.exec(source)) { |
| 69 | if (!temp_patterns_for_checking.includes(found[1])) { |
| 70 | temp_patterns_for_checking.push(found[1]) |
| 71 | if (item.type === 'link') { |
| 72 | found[1] = decodeURIComponent(found[1]) |
| 73 | } |
| 74 | temp_patterns_list.push({ |
| 75 | type: item.type, |
| 76 | matched: found[1] |
| 77 | }) |
| 78 | } |
| 79 | } |
| 80 | }) |
| 81 | } |
| 82 | return temp_patterns_list |
| 83 | } catch (err) { |
| 84 | helper.verbose && console.log(err) |
| 85 | return [] |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | export default{ |
| 90 | extract_patterns, |
| 91 | extract_metadata |
| 92 | } |
| 93 |