Merge pull request 'feat: add-test and some fixes' (#1) from feat/add-test into master
Reviewed-on: #1
This commit is contained in:
commit
9647ab42ed
|
|
@ -17,6 +17,7 @@ Inspired by [DatMusic](https://github.com/alashow/datmusic-api), [MyFreeMp3](htt
|
||||||
*TODO*
|
*TODO*
|
||||||
|
|
||||||
* Dowload all songs in a folder
|
* Dowload all songs in a folder
|
||||||
|
* Add Spotify metadatas in the m3u8 file
|
||||||
* Add proper command line options
|
* Add proper command line options
|
||||||
* Rewrite this as a plugin for Ampache
|
* Rewrite this as a plugin for Ampache
|
||||||
* Rewrite this as a plugin for Funkwhale
|
* Rewrite this as a plugin for Funkwhale
|
||||||
|
|
@ -32,3 +33,11 @@ Inspired by [DatMusic](https://github.com/alashow/datmusic-api), [MyFreeMp3](htt
|
||||||
|
|
||||||
1. Install nodejs/npm
|
1. Install nodejs/npm
|
||||||
1. `npx vk-music-dl https://open.spotify.com/playlist/6LgeEhc97Azxq6sinJQt6w`
|
1. `npx vk-music-dl https://open.spotify.com/playlist/6LgeEhc97Azxq6sinJQt6w`
|
||||||
|
|
||||||
|
## Usage as library
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const {main} = require('vk-music-dl/lib');
|
||||||
|
|
||||||
|
main('https://open.spotify.com/playlist/6LgeEhc97Azxq6sinJQt6w');
|
||||||
|
```
|
||||||
|
|
|
||||||
144
index.js
144
index.js
|
|
@ -1,148 +1,6 @@
|
||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
'use strict';
|
'use strict';
|
||||||
const Cheerio = require('cheerio');
|
|
||||||
const { promises: fs, createWriteStream } = require('fs');
|
|
||||||
const debug = require('debug')('vk-music-dl');
|
|
||||||
const vm = require('vm');
|
|
||||||
const Promise = require('bluebird');
|
|
||||||
const _ = require('lodash');
|
|
||||||
const leven = require('leven');
|
|
||||||
const { URL, URLSearchParams } = require('url');
|
|
||||||
const fetch = require('node-fetch');
|
|
||||||
const m3u = require('m3u');
|
|
||||||
const promisepipe = require('promisepipe');
|
|
||||||
require('dotenv').config();
|
require('dotenv').config();
|
||||||
|
const { main } = require('./lib');
|
||||||
|
|
||||||
async function getSpotifyPlaylist(playListUrl){
|
|
||||||
const spotifyPlaylistPageContent = await fetch(playListUrl, {
|
|
||||||
headers: {
|
|
||||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.then(res => res.text())
|
|
||||||
|
|
||||||
const $ = Cheerio.load(spotifyPlaylistPageContent);
|
|
||||||
|
|
||||||
let config;
|
|
||||||
$('script#config').each(function(i, elem){
|
|
||||||
config = JSON.parse($(this).html());
|
|
||||||
});
|
|
||||||
debug('config=%O', config);
|
|
||||||
|
|
||||||
const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1];
|
|
||||||
|
|
||||||
const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, {
|
|
||||||
headers: {
|
|
||||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
|
|
||||||
Authorization: `Bearer ${config.accessToken}`
|
|
||||||
}
|
|
||||||
}).then(res => res.json());
|
|
||||||
|
|
||||||
return playlist;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function searchOnVkMusic(query){
|
|
||||||
const url = new URL('https://api.vk.com/method/audio.search');
|
|
||||||
url.search = new URLSearchParams({
|
|
||||||
v: '5.71',
|
|
||||||
access_token: process.env.ACCESS_TOKEN,
|
|
||||||
q: query,
|
|
||||||
count: 200,
|
|
||||||
}).toString();
|
|
||||||
|
|
||||||
return await fetch(url, {
|
|
||||||
headers: {
|
|
||||||
'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)',
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.then(res => res.json())
|
|
||||||
.then(responseBody => {
|
|
||||||
if(responseBody.error){
|
|
||||||
throw _.assign(responseBody.error, new Error(responseBody.error.error_msg));
|
|
||||||
} else {
|
|
||||||
return responseBody.response.items;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function searchOnMyFreeMp3(query){
|
|
||||||
const url = new URL('https://myfreemp3cc.com/api/search.php?callback=jQuery666');
|
|
||||||
|
|
||||||
return await fetch(url, {
|
|
||||||
method: 'POST',
|
|
||||||
body: new URLSearchParams({
|
|
||||||
q: query,
|
|
||||||
page: 0,
|
|
||||||
}),
|
|
||||||
})
|
|
||||||
.then(res => res.text())
|
|
||||||
.then(page => {
|
|
||||||
debug('page=%O', page);
|
|
||||||
return page;
|
|
||||||
})
|
|
||||||
.then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response }))
|
|
||||||
.then(items => _.filter(items, item => !_.isString(item)));
|
|
||||||
}
|
|
||||||
|
|
||||||
function matchScore(spotifyMetas, vkmusicMetas){
|
|
||||||
const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase();
|
|
||||||
const originalTitle = spotifyMetas.name.toLowerCase();
|
|
||||||
const originalDuration = Math.round(spotifyMetas.duration_ms/1000);
|
|
||||||
|
|
||||||
const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length));
|
|
||||||
const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length));
|
|
||||||
const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0
|
|
||||||
debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore);
|
|
||||||
return matchArtistScore + matchTitleScore + matchDurationScore;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function generateM3U8Playlist(filesWithMetas){
|
|
||||||
const m3uWriter = m3u.extendedWriter();
|
|
||||||
filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`));
|
|
||||||
await fs.writeFile(`playlist.m3u8`, m3uWriter.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main(playlistUrl){
|
|
||||||
const playlist = await getSpotifyPlaylist(playlistUrl);
|
|
||||||
const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => {
|
|
||||||
const artistNames = _.map(track.artists, 'name').join(', ');
|
|
||||||
debug('%s - %s', track.name, artistNames);
|
|
||||||
const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`);
|
|
||||||
debug('items=%O', items);
|
|
||||||
const bestMatch = _.chain(items).map((item) => {
|
|
||||||
item.score = matchScore(track, item);
|
|
||||||
return item;
|
|
||||||
}).filter(item => item.score && item.url).sortBy('score').last().value();
|
|
||||||
if(!bestMatch){
|
|
||||||
console.log(`You are on your own for ${track.name} - ${artistNames}`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`;
|
|
||||||
debug('bestMatch=%O', bestMatch);
|
|
||||||
try {
|
|
||||||
await fs.access(bestMatch.path).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems
|
|
||||||
await fetch(bestMatch.url).then(async res => {
|
|
||||||
await promisepipe(res.body, createWriteStream(bestMatch.path));
|
|
||||||
});
|
|
||||||
debug('Done downloading');
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Download failed for ', bestMatch);
|
|
||||||
console.error(err);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return bestMatch;
|
|
||||||
}, {concurrency: 1});
|
|
||||||
|
|
||||||
await generateM3U8Playlist(_.compact(vkPlaylist));
|
|
||||||
}
|
|
||||||
|
|
||||||
main(process.argv[2]).catch(err => console.error(err));
|
main(process.argv[2]).catch(err => console.error(err));
|
||||||
|
|
||||||
async function test(query){
|
|
||||||
const response = await searchOnMyFreeMp3(query);
|
|
||||||
console.log(response);
|
|
||||||
}
|
|
||||||
|
|
||||||
//test(process.argv[2]);
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,155 @@
|
||||||
|
'use strict';
|
||||||
|
const Cheerio = require('cheerio');
|
||||||
|
const { promises: fs, createWriteStream } = require('fs');
|
||||||
|
const debug = require('debug')('vk-music-dl');
|
||||||
|
const vm = require('vm');
|
||||||
|
const Promise = require('bluebird');
|
||||||
|
const _ = require('lodash');
|
||||||
|
const leven = require('leven');
|
||||||
|
const { URL, URLSearchParams } = require('url');
|
||||||
|
const fetch = require('node-fetch');
|
||||||
|
const m3u = require('m3u');
|
||||||
|
const promisepipe = require('promisepipe');
|
||||||
|
const path = require('path');
|
||||||
|
require('dotenv').config();
|
||||||
|
|
||||||
|
const userAgentString = 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0';
|
||||||
|
|
||||||
|
async function getSpotifyPlaylist(playListUrl){
|
||||||
|
const spotifyPlaylistPageContent = await fetch(playListUrl, {
|
||||||
|
headers: {
|
||||||
|
'User-Agent': userAgentString,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.then(res => res.text())
|
||||||
|
|
||||||
|
const $ = Cheerio.load(spotifyPlaylistPageContent);
|
||||||
|
|
||||||
|
let config;
|
||||||
|
$('script#config').each(function(i, elem){
|
||||||
|
config = JSON.parse($(this).html());
|
||||||
|
});
|
||||||
|
debug('config=%O', config);
|
||||||
|
|
||||||
|
const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1];
|
||||||
|
|
||||||
|
const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, {
|
||||||
|
headers: {
|
||||||
|
'User-Agent': userAgentString,
|
||||||
|
Authorization: `Bearer ${config.accessToken}`
|
||||||
|
}
|
||||||
|
}).then(res => res.json());
|
||||||
|
|
||||||
|
return playlist;
|
||||||
|
}
|
||||||
|
exports.getSpotifyPlaylist = getSpotifyPlaylist;
|
||||||
|
|
||||||
|
async function searchOnVkMusic(query){
|
||||||
|
const url = new URL('https://api.vk.com/method/audio.search');
|
||||||
|
url.search = new URLSearchParams({
|
||||||
|
v: '5.71',
|
||||||
|
access_token: process.env.ACCESS_TOKEN,
|
||||||
|
q: query,
|
||||||
|
count: 200,
|
||||||
|
}).toString();
|
||||||
|
|
||||||
|
return await fetch(url, {
|
||||||
|
headers: {
|
||||||
|
'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)',
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.then(res => res.json())
|
||||||
|
.then(responseBody => {
|
||||||
|
if(responseBody.error){
|
||||||
|
throw _.assign(responseBody.error, new Error(responseBody.error.error_msg));
|
||||||
|
} else {
|
||||||
|
return responseBody.response.items;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
exports.searchOnVkMusic = searchOnVkMusic;
|
||||||
|
|
||||||
|
async function searchOnMyFreeMp3(query){
|
||||||
|
const url = new URL('https://myfreemp3.vip/api/search.php?callback=jQuery666');
|
||||||
|
|
||||||
|
return await fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'User-Agent': userAgentString,
|
||||||
|
'X-Requested-With': 'XMLHttpRequest',
|
||||||
|
Accept: 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
|
||||||
|
'Accept-Language': 'fr-FR,fr;q=0.5',
|
||||||
|
Referer: 'https://myfreemp3.vip/',
|
||||||
|
Origin: 'https://myfreemp3.vip',
|
||||||
|
//Cookie: '__cfduid=d1821f08faee82ed9d4bf93b1b5fb9d3e1601196911; musicLang=en'
|
||||||
|
},
|
||||||
|
body: new URLSearchParams({
|
||||||
|
q: query,
|
||||||
|
page: 0,
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
.then(res => res.text())
|
||||||
|
.then(page => {
|
||||||
|
debug('page=%O', page);
|
||||||
|
return page;
|
||||||
|
})
|
||||||
|
.then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response }))
|
||||||
|
.then(items => _.filter(items, item => !_.isString(item)));
|
||||||
|
}
|
||||||
|
exports.searchOnMyFreeMp3 = searchOnMyFreeMp3;
|
||||||
|
|
||||||
|
function matchScore(spotifyMetas, vkmusicMetas){
|
||||||
|
const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase();
|
||||||
|
const originalTitle = spotifyMetas.name.toLowerCase();
|
||||||
|
const originalDuration = Math.round(spotifyMetas.duration_ms/1000);
|
||||||
|
|
||||||
|
const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length));
|
||||||
|
const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length));
|
||||||
|
const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0
|
||||||
|
debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore);
|
||||||
|
return matchArtistScore + matchTitleScore + matchDurationScore;
|
||||||
|
}
|
||||||
|
exports.matchScore = matchScore;
|
||||||
|
|
||||||
|
function generateM3U8Playlist(filesWithMetas){
|
||||||
|
const m3uWriter = m3u.extendedWriter();
|
||||||
|
filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`));
|
||||||
|
return m3uWriter.toString();
|
||||||
|
}
|
||||||
|
exports.generateM3U8Playlist = generateM3U8Playlist;
|
||||||
|
|
||||||
|
exports.main = async function main(playlistUrl){
|
||||||
|
const playlist = await getSpotifyPlaylist(playlistUrl);
|
||||||
|
await Promise.resolve(fs.mkdir(playlist.name)).catch({code: 'EEXIST'}, () => {});
|
||||||
|
const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => {
|
||||||
|
const artistNames = _.map(track.artists, 'name').join(', ');
|
||||||
|
debug('%s - %s', track.name, artistNames);
|
||||||
|
const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`);
|
||||||
|
debug('items=%O', items);
|
||||||
|
const bestMatch = _.chain(items).map((item) => {
|
||||||
|
item.score = matchScore(track, item);
|
||||||
|
return item;
|
||||||
|
}).filter(item => item.score && item.url).sortBy('score').last().value();
|
||||||
|
if(!bestMatch){
|
||||||
|
console.log(`You are on your own for ${track.name} - ${artistNames}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`;
|
||||||
|
debug('bestMatch=%O', bestMatch);
|
||||||
|
try {
|
||||||
|
await fs.access(path.join(playlist.name, bestMatch.path)).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems
|
||||||
|
await fetch(bestMatch.url).then(async res => {
|
||||||
|
await promisepipe(res.body, createWriteStream(path.join(playlist.name, bestMatch.path)));
|
||||||
|
});
|
||||||
|
debug('Done downloading');
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Download failed for ', bestMatch);
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return bestMatch;
|
||||||
|
}, {concurrency: 1});
|
||||||
|
|
||||||
|
await fs.writeFile(path.join(playlist.name, `playlist.m3u8`), generateM3U8Playlist(_.compact(vkPlaylist)));
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -4,7 +4,7 @@
|
||||||
"description": "Download Spotify playlist as mp3 from VKontakte",
|
"description": "Download Spotify playlist as mp3 from VKontakte",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "echo \"Error: no test specified\" && exit 1"
|
"test": "mocha"
|
||||||
},
|
},
|
||||||
"bin": {
|
"bin": {
|
||||||
"vk-music-dl": "./index.js"
|
"vk-music-dl": "./index.js"
|
||||||
|
|
@ -21,6 +21,7 @@
|
||||||
"downloader"
|
"downloader"
|
||||||
],
|
],
|
||||||
"files": [
|
"files": [
|
||||||
|
"lib",
|
||||||
"index.js",
|
"index.js",
|
||||||
"README.md"
|
"README.md"
|
||||||
],
|
],
|
||||||
|
|
@ -36,5 +37,7 @@
|
||||||
"node-fetch": "^2.6.0",
|
"node-fetch": "^2.6.0",
|
||||||
"promisepipe": "^3.0.0"
|
"promisepipe": "^3.0.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {}
|
"devDependencies": {
|
||||||
|
"mocha": "^8.1.3"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
'use strict';
|
||||||
|
const assert = require('assert');
|
||||||
|
const {getSpotifyPlaylist} = require('../lib');
|
||||||
|
|
||||||
|
describe('getSpotifyPlaylist', function(){
|
||||||
|
|
||||||
|
it('acquire and parse playlist from Spotify', async function(){
|
||||||
|
const playlist = await getSpotifyPlaylist('https://open.spotify.com/playlist/27OFA3NDwoLWtmdMvowA3S');
|
||||||
|
assert.strictEqual(playlist.name, 'The Best Yoga & Cool-Down Songs');
|
||||||
|
assert.strictEqual(playlist.type, 'playlist');
|
||||||
|
assert.strictEqual(playlist.tracks.items.length, 10);
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
Loading…
Reference in New Issue