'use strict'; const Cheerio = require('cheerio'); const { promises: fs, createWriteStream } = require('fs'); const debug = require('debug')('vk-music-dl'); const verboseDebug = require('debug')('vk-music-dl:debug'); const vm = require('vm'); const Promise = require('bluebird'); const _ = require('lodash'); const leven = require('leven'); const { URL, URLSearchParams } = require('url'); const fetch = require('node-fetch'); const m3u = require('m3u'); const promisepipe = require('promisepipe'); const path = require('path'); require('dotenv').config(); const userAgentString = 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0'; async function getSpotifyPlaylist(playListUrl){ const spotifyPlaylistPageContent = await fetch(playListUrl, { headers: { 'User-Agent': userAgentString, } }) .then(res => res.text()) const $ = Cheerio.load(spotifyPlaylistPageContent); let config; $('script#config').each(function(i, elem){ config = JSON.parse($(this).html()); }); verboseDebug('config=%O', config); const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1]; const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, { headers: { 'User-Agent': userAgentString, Authorization: `Bearer ${config.accessToken}` } }).then(res => res.json()); verboseDebug('playlist=%O', playlist); return playlist; } exports.getSpotifyPlaylist = getSpotifyPlaylist; async function searchOnVkMusic(query){ const url = new URL('https://api.vk.com/method/audio.search'); url.search = new URLSearchParams({ v: '5.71', access_token: process.env.ACCESS_TOKEN, q: query, count: 200, }).toString(); return await fetch(url, { headers: { 'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)', } }) .then(res => res.json()) .then(responseBody => { if(responseBody.error){ throw _.assign(responseBody.error, new Error(responseBody.error.error_msg)); } else { return responseBody.response.items; } }); } exports.searchOnVkMusic = searchOnVkMusic; async function searchOnMyFreeMp3(query){ const url = new URL('https://myfreemp3.vip/api/search.php?callback=jQuery666'); return await fetch(url, { method: 'POST', headers: { 'User-Agent': userAgentString, 'X-Requested-With': 'XMLHttpRequest', Accept: 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01', 'Accept-Language': 'fr-FR,fr;q=0.5', Referer: 'https://myfreemp3.vip/', Origin: 'https://myfreemp3.vip', //Cookie: '__cfduid=d1821f08faee82ed9d4bf93b1b5fb9d3e1601196911; musicLang=en' }, body: new URLSearchParams({ q: query, page: 0, }), }) .then(res => res.text()) .then(page => { verboseDebug('page=%O', page); return page; }) .then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response })) .then(items => _.filter(items, item => !_.isString(item))); } exports.searchOnMyFreeMp3 = searchOnMyFreeMp3; function matchScore(spotifyMetas, vkmusicMetas){ const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase(); const originalTitle = spotifyMetas.name.toLowerCase(); const originalDuration = Math.round(spotifyMetas.duration_ms/1000); const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length)); const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length)); const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0 verboseDebug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore); return matchArtistScore + matchTitleScore + matchDurationScore; } exports.matchScore = matchScore; function generateM3U8Playlist(playlist){ const m3uWriter = m3u.extendedWriter(); m3uWriter.comment(`VK-MUSIC-DL SPOTIFYURL ${playlist.external_urls.spotify}`); playlist.tracks.items.forEach(item => { if(item.destPath){ m3uWriter.file(item.destPath, item.vkBestMatch.duration, `${item.vkBestMatch.artist} - ${item.vkBestMatch.title}`); } else { const artistNames = _.map(item.track.artists, 'name').join(', '); m3uWriter.comment(`VK-MUSIC-DL Missing song ${item.track.name} - ${artistNames}`); } }); return m3uWriter.toString(); } exports.generateM3U8Playlist = generateM3U8Playlist; exports.main = async function main(playlistUrl){ const playlist = await getSpotifyPlaylist(playlistUrl); const playlistPath = playlist.name.replace(/\/|\\/g, ''); await Promise.resolve(fs.mkdir(playlistPath)).catch({code: 'EEXIST'}, () => {}); playlist.tracks.items = await Promise.map(playlist.tracks.items, async (item) => { const artistNames = _.map(item.track.artists, 'name').join(', '); debug('%s - %s', item.track.name, artistNames); const myFreeMp3Results = await searchOnMyFreeMp3(`${artistNames} ${item.track.name}`); verboseDebug('myFreeMp3Results=%O', myFreeMp3Results); const bestMatch = _.chain(myFreeMp3Results).map((result) => { result.score = matchScore(item.track, result); return result; }).filter(result => result.score && result.url).sortBy('score').last().value(); if(bestMatch){ verboseDebug('bestMatch=%O', bestMatch); item.vkBestMatch = bestMatch; const destPath = `${bestMatch.artist} - ${bestMatch.title}.mp3`.replace(/\/|\\/g, ''); try { await fs.access(path.join(playlistPath, destPath)).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems await fetch(bestMatch.url).then(async res => { // TODO check the mime type here await promisepipe(res.body, createWriteStream(path.join(playlistPath, destPath))); }); debug('Done downloading'); }); item.destPath = destPath; } catch (err) { console.error('Download failed for ', bestMatch); console.error(err); } } else { console.log(`You are on your own for ${item.track.name} - ${artistNames}`); } return item; }, {concurrency: 1}); await fs.writeFile(path.join(playlistPath, `playlist.m3u8`), generateM3U8Playlist(playlist)); }