feat: add-test and some fixes #1

Merged
hugopoi merged 4 commits from feat/add-test into master 2020-09-27 11:41:19 +02:00
2 changed files with 144 additions and 143 deletions
Showing only changes of commit 81aa88f889 - Show all commits

144
index.js
View File

@ -1,148 +1,6 @@
#!/usr/bin/env node #!/usr/bin/env node
'use strict'; 'use strict';
const Cheerio = require('cheerio');
const { promises: fs, createWriteStream } = require('fs');
const debug = require('debug')('vk-music-dl');
const vm = require('vm');
const Promise = require('bluebird');
const _ = require('lodash');
const leven = require('leven');
const { URL, URLSearchParams } = require('url');
const fetch = require('node-fetch');
const m3u = require('m3u');
const promisepipe = require('promisepipe');
require('dotenv').config(); require('dotenv').config();
const { main } = require('./lib');
async function getSpotifyPlaylist(playListUrl){
const spotifyPlaylistPageContent = await fetch(playListUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
}
})
.then(res => res.text())
const $ = Cheerio.load(spotifyPlaylistPageContent);
let config;
$('script#config').each(function(i, elem){
config = JSON.parse($(this).html());
});
debug('config=%O', config);
const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1];
const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, {
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
Authorization: `Bearer ${config.accessToken}`
}
}).then(res => res.json());
return playlist;
}
async function searchOnVkMusic(query){
const url = new URL('https://api.vk.com/method/audio.search');
url.search = new URLSearchParams({
v: '5.71',
access_token: process.env.ACCESS_TOKEN,
q: query,
count: 200,
}).toString();
return await fetch(url, {
headers: {
'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)',
}
})
.then(res => res.json())
.then(responseBody => {
if(responseBody.error){
throw _.assign(responseBody.error, new Error(responseBody.error.error_msg));
} else {
return responseBody.response.items;
}
});
}
async function searchOnMyFreeMp3(query){
const url = new URL('https://myfreemp3cc.com/api/search.php?callback=jQuery666');
return await fetch(url, {
method: 'POST',
body: new URLSearchParams({
q: query,
page: 0,
}),
})
.then(res => res.text())
.then(page => {
debug('page=%O', page);
return page;
})
.then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response }))
.then(items => _.filter(items, item => !_.isString(item)));
}
function matchScore(spotifyMetas, vkmusicMetas){
const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase();
const originalTitle = spotifyMetas.name.toLowerCase();
const originalDuration = Math.round(spotifyMetas.duration_ms/1000);
const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length));
const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length));
const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0
debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore);
return matchArtistScore + matchTitleScore + matchDurationScore;
}
async function generateM3U8Playlist(filesWithMetas){
const m3uWriter = m3u.extendedWriter();
filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`));
await fs.writeFile(`playlist.m3u8`, m3uWriter.toString());
}
async function main(playlistUrl){
const playlist = await getSpotifyPlaylist(playlistUrl);
const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => {
const artistNames = _.map(track.artists, 'name').join(', ');
debug('%s - %s', track.name, artistNames);
const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`);
debug('items=%O', items);
const bestMatch = _.chain(items).map((item) => {
item.score = matchScore(track, item);
return item;
}).filter(item => item.score && item.url).sortBy('score').last().value();
if(!bestMatch){
console.log(`You are on your own for ${track.name} - ${artistNames}`);
return;
}
bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`;
debug('bestMatch=%O', bestMatch);
try {
await fs.access(bestMatch.path).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems
await fetch(bestMatch.url).then(async res => {
await promisepipe(res.body, createWriteStream(bestMatch.path));
});
debug('Done downloading');
});
} catch (err) {
console.error('Download failed for ', bestMatch);
console.error(err);
return null;
}
return bestMatch;
}, {concurrency: 1});
await generateM3U8Playlist(_.compact(vkPlaylist));
}
main(process.argv[2]).catch(err => console.error(err)); main(process.argv[2]).catch(err => console.error(err));
async function test(query){
const response = await searchOnMyFreeMp3(query);
console.log(response);
}
//test(process.argv[2]);

143
lib/index.js Normal file
View File

@ -0,0 +1,143 @@
'use strict';
const Cheerio = require('cheerio');
const { promises: fs, createWriteStream } = require('fs');
const debug = require('debug')('vk-music-dl');
const vm = require('vm');
const Promise = require('bluebird');
const _ = require('lodash');
const leven = require('leven');
const { URL, URLSearchParams } = require('url');
const fetch = require('node-fetch');
const m3u = require('m3u');
const promisepipe = require('promisepipe');
require('dotenv').config();
async function getSpotifyPlaylist(playListUrl){
const spotifyPlaylistPageContent = await fetch(playListUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
}
})
.then(res => res.text())
const $ = Cheerio.load(spotifyPlaylistPageContent);
let config;
$('script#config').each(function(i, elem){
config = JSON.parse($(this).html());
});
debug('config=%O', config);
const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1];
const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, {
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
Authorization: `Bearer ${config.accessToken}`
}
}).then(res => res.json());
return playlist;
}
exports.getSpotifyPlaylist = getSpotifyPlaylist;
async function searchOnVkMusic(query){
const url = new URL('https://api.vk.com/method/audio.search');
url.search = new URLSearchParams({
v: '5.71',
access_token: process.env.ACCESS_TOKEN,
q: query,
count: 200,
}).toString();
return await fetch(url, {
headers: {
'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)',
}
})
.then(res => res.json())
.then(responseBody => {
if(responseBody.error){
throw _.assign(responseBody.error, new Error(responseBody.error.error_msg));
} else {
return responseBody.response.items;
}
});
}
exports.searchOnVkMusic = searchOnVkMusic;
async function searchOnMyFreeMp3(query){
const url = new URL('https://myfreemp3.vip/api/search.php?callback=jQuery666');
return await fetch(url, {
method: 'POST',
body: new URLSearchParams({
q: query,
page: 0,
}),
})
.then(res => res.text())
.then(page => {
debug('page=%O', page);
return page;
})
.then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response }))
.then(items => _.filter(items, item => !_.isString(item)));
}
exports.searchOnMyFreeMp3 = searchOnMyFreeMp3;
function matchScore(spotifyMetas, vkmusicMetas){
const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase();
const originalTitle = spotifyMetas.name.toLowerCase();
const originalDuration = Math.round(spotifyMetas.duration_ms/1000);
const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length));
const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length));
const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0
debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore);
return matchArtistScore + matchTitleScore + matchDurationScore;
}
exports.matchScore = matchScore;
function generateM3U8Playlist(filesWithMetas){
const m3uWriter = m3u.extendedWriter();
filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`));
return m3uWriter.toString();
}
exports.generateM3U8Playlist = generateM3U8Playlist;
exports.main = async function main(playlistUrl){
const playlist = await getSpotifyPlaylist(playlistUrl);
const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => {
const artistNames = _.map(track.artists, 'name').join(', ');
debug('%s - %s', track.name, artistNames);
const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`);
debug('items=%O', items);
const bestMatch = _.chain(items).map((item) => {
item.score = matchScore(track, item);
return item;
}).filter(item => item.score && item.url).sortBy('score').last().value();
if(!bestMatch){
console.log(`You are on your own for ${track.name} - ${artistNames}`);
return;
}
bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`;
debug('bestMatch=%O', bestMatch);
try {
await fs.access(bestMatch.path).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems
await fetch(bestMatch.url).then(async res => {
await promisepipe(res.body, createWriteStream(bestMatch.path));
});
debug('Done downloading');
});
} catch (err) {
console.error('Download failed for ', bestMatch);
console.error(err);
return null;
}
return bestMatch;
}, {concurrency: 1});
await fs.writeFile(`playlist.m3u8`, generateM3U8Playlist(_.compact(vkPlaylist)));
}