diff --git a/index.js b/index.js index b5aab4e..819680e 100755 --- a/index.js +++ b/index.js @@ -1,148 +1,6 @@ #!/usr/bin/env node 'use strict'; -const Cheerio = require('cheerio'); -const { promises: fs, createWriteStream } = require('fs'); -const debug = require('debug')('vk-music-dl'); -const vm = require('vm'); -const Promise = require('bluebird'); -const _ = require('lodash'); -const leven = require('leven'); -const { URL, URLSearchParams } = require('url'); -const fetch = require('node-fetch'); -const m3u = require('m3u'); -const promisepipe = require('promisepipe'); require('dotenv').config(); - - -async function getSpotifyPlaylist(playListUrl){ - const spotifyPlaylistPageContent = await fetch(playListUrl, { - headers: { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0', - } - }) - .then(res => res.text()) - - const $ = Cheerio.load(spotifyPlaylistPageContent); - - let config; - $('script#config').each(function(i, elem){ - config = JSON.parse($(this).html()); - }); - debug('config=%O', config); - - const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1]; - - const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, { - headers: { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0', - Authorization: `Bearer ${config.accessToken}` - } - }).then(res => res.json()); - - return playlist; -} - -async function searchOnVkMusic(query){ - const url = new URL('https://api.vk.com/method/audio.search'); - url.search = new URLSearchParams({ - v: '5.71', - access_token: process.env.ACCESS_TOKEN, - q: query, - count: 200, - }).toString(); - - return await fetch(url, { - headers: { - 'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)', - } - }) - .then(res => res.json()) - .then(responseBody => { - if(responseBody.error){ - throw _.assign(responseBody.error, new Error(responseBody.error.error_msg)); - } else { - return responseBody.response.items; - } - }); -} - -async function searchOnMyFreeMp3(query){ - const url = new URL('https://myfreemp3cc.com/api/search.php?callback=jQuery666'); - - return await fetch(url, { - method: 'POST', - body: new URLSearchParams({ - q: query, - page: 0, - }), - }) - .then(res => res.text()) - .then(page => { - debug('page=%O', page); - return page; - }) - .then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response })) - .then(items => _.filter(items, item => !_.isString(item))); -} - -function matchScore(spotifyMetas, vkmusicMetas){ - const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase(); - const originalTitle = spotifyMetas.name.toLowerCase(); - const originalDuration = Math.round(spotifyMetas.duration_ms/1000); - - const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length)); - const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length)); - const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0 - debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore); - return matchArtistScore + matchTitleScore + matchDurationScore; -} - -async function generateM3U8Playlist(filesWithMetas){ - const m3uWriter = m3u.extendedWriter(); - filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`)); - await fs.writeFile(`playlist.m3u8`, m3uWriter.toString()); -} - -async function main(playlistUrl){ - const playlist = await getSpotifyPlaylist(playlistUrl); - const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => { - const artistNames = _.map(track.artists, 'name').join(', '); - debug('%s - %s', track.name, artistNames); - const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`); - debug('items=%O', items); - const bestMatch = _.chain(items).map((item) => { - item.score = matchScore(track, item); - return item; - }).filter(item => item.score && item.url).sortBy('score').last().value(); - if(!bestMatch){ - console.log(`You are on your own for ${track.name} - ${artistNames}`); - return; - } - bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`; - debug('bestMatch=%O', bestMatch); - try { - await fs.access(bestMatch.path).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems - await fetch(bestMatch.url).then(async res => { - await promisepipe(res.body, createWriteStream(bestMatch.path)); - }); - debug('Done downloading'); - }); - } catch (err) { - console.error('Download failed for ', bestMatch); - console.error(err); - return null; - } - return bestMatch; - }, {concurrency: 1}); - - await generateM3U8Playlist(_.compact(vkPlaylist)); -} +const { main } = require('./lib'); main(process.argv[2]).catch(err => console.error(err)); - -async function test(query){ - const response = await searchOnMyFreeMp3(query); - console.log(response); -} - -//test(process.argv[2]); diff --git a/lib/index.js b/lib/index.js new file mode 100644 index 0000000..5c6d6df --- /dev/null +++ b/lib/index.js @@ -0,0 +1,143 @@ +'use strict'; +const Cheerio = require('cheerio'); +const { promises: fs, createWriteStream } = require('fs'); +const debug = require('debug')('vk-music-dl'); +const vm = require('vm'); +const Promise = require('bluebird'); +const _ = require('lodash'); +const leven = require('leven'); +const { URL, URLSearchParams } = require('url'); +const fetch = require('node-fetch'); +const m3u = require('m3u'); +const promisepipe = require('promisepipe'); +require('dotenv').config(); + + +async function getSpotifyPlaylist(playListUrl){ + const spotifyPlaylistPageContent = await fetch(playListUrl, { + headers: { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0', + } + }) + .then(res => res.text()) + + const $ = Cheerio.load(spotifyPlaylistPageContent); + + let config; + $('script#config').each(function(i, elem){ + config = JSON.parse($(this).html()); + }); + debug('config=%O', config); + + const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1]; + + const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, { + headers: { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0', + Authorization: `Bearer ${config.accessToken}` + } + }).then(res => res.json()); + + return playlist; +} +exports.getSpotifyPlaylist = getSpotifyPlaylist; + +async function searchOnVkMusic(query){ + const url = new URL('https://api.vk.com/method/audio.search'); + url.search = new URLSearchParams({ + v: '5.71', + access_token: process.env.ACCESS_TOKEN, + q: query, + count: 200, + }).toString(); + + return await fetch(url, { + headers: { + 'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)', + } + }) + .then(res => res.json()) + .then(responseBody => { + if(responseBody.error){ + throw _.assign(responseBody.error, new Error(responseBody.error.error_msg)); + } else { + return responseBody.response.items; + } + }); +} +exports.searchOnVkMusic = searchOnVkMusic; + +async function searchOnMyFreeMp3(query){ + const url = new URL('https://myfreemp3.vip/api/search.php?callback=jQuery666'); + + return await fetch(url, { + method: 'POST', + body: new URLSearchParams({ + q: query, + page: 0, + }), + }) + .then(res => res.text()) + .then(page => { + debug('page=%O', page); + return page; + }) + .then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response })) + .then(items => _.filter(items, item => !_.isString(item))); +} +exports.searchOnMyFreeMp3 = searchOnMyFreeMp3; + +function matchScore(spotifyMetas, vkmusicMetas){ + const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase(); + const originalTitle = spotifyMetas.name.toLowerCase(); + const originalDuration = Math.round(spotifyMetas.duration_ms/1000); + + const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length)); + const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length)); + const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0 + debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore); + return matchArtistScore + matchTitleScore + matchDurationScore; +} +exports.matchScore = matchScore; + +function generateM3U8Playlist(filesWithMetas){ + const m3uWriter = m3u.extendedWriter(); + filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`)); + return m3uWriter.toString(); +} +exports.generateM3U8Playlist = generateM3U8Playlist; + +exports.main = async function main(playlistUrl){ + const playlist = await getSpotifyPlaylist(playlistUrl); + const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => { + const artistNames = _.map(track.artists, 'name').join(', '); + debug('%s - %s', track.name, artistNames); + const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`); + debug('items=%O', items); + const bestMatch = _.chain(items).map((item) => { + item.score = matchScore(track, item); + return item; + }).filter(item => item.score && item.url).sortBy('score').last().value(); + if(!bestMatch){ + console.log(`You are on your own for ${track.name} - ${artistNames}`); + return; + } + bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`; + debug('bestMatch=%O', bestMatch); + try { + await fs.access(bestMatch.path).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems + await fetch(bestMatch.url).then(async res => { + await promisepipe(res.body, createWriteStream(bestMatch.path)); + }); + debug('Done downloading'); + }); + } catch (err) { + console.error('Download failed for ', bestMatch); + console.error(err); + return null; + } + return bestMatch; + }, {concurrency: 1}); + + await fs.writeFile(`playlist.m3u8`, generateM3U8Playlist(_.compact(vkPlaylist))); +}