feat: m3u8 have some metadata about missing songs and spotify playlist id

This commit is contained in:
HugoPoi 2020-10-27 23:33:10 +01:00
parent 42692034ea
commit 9e9f280f15
2 changed files with 50 additions and 37 deletions

View File

@ -17,17 +17,18 @@ Inspired by [DatMusic](https://github.com/alashow/datmusic-api), [MyFreeMp3](htt
*TODO* *TODO*
* ~~Dowload all songs in a folder~~ * ~~Dowload all songs in a folder~~
* Add Spotify metadatas in the m3u8 file * ~~Add Spotify metadatas in the m3u8 file~~
* Add proper command line options * Add proper command line options
* Add CLI progress info * Add CLI progress info
* Having placeholder for files that can't be downloaded, maybe as * ~~Having placeholder for files that can't be downloaded, maybe as
comment in the m3u8 file comment in the m3u8 file~~
* Having a option to retry/resync download mp3 based on the m3u8 file * Having a option to retry/resync download mp3 based on the m3u8 file
* Check the mime type of receive files because some are html files * Check the mime type of receive files because some are html files
instead of mp3 WTF !! instead of mp3 WTF !!
* Rewrite this as a plugin for Ampache * Rewrite this as a plugin for Ampache
* Rewrite this as a plugin for Funkwhale * Rewrite this as a plugin for Funkwhale
* Add MusicBrain ID3Tag on file * Add MusicBrain ID3Tag on file
* Have a fallback for downloading song with youtube-dl :-P
## Goals ## Goals

View File

@ -2,6 +2,7 @@
const Cheerio = require('cheerio'); const Cheerio = require('cheerio');
const { promises: fs, createWriteStream } = require('fs'); const { promises: fs, createWriteStream } = require('fs');
const debug = require('debug')('vk-music-dl'); const debug = require('debug')('vk-music-dl');
const verboseDebug = require('debug')('vk-music-dl:debug');
const vm = require('vm'); const vm = require('vm');
const Promise = require('bluebird'); const Promise = require('bluebird');
const _ = require('lodash'); const _ = require('lodash');
@ -29,7 +30,7 @@ async function getSpotifyPlaylist(playListUrl){
$('script#config').each(function(i, elem){ $('script#config').each(function(i, elem){
config = JSON.parse($(this).html()); config = JSON.parse($(this).html());
}); });
debug('config=%O', config); verboseDebug('config=%O', config);
const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1]; const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1];
@ -40,6 +41,7 @@ async function getSpotifyPlaylist(playListUrl){
} }
}).then(res => res.json()); }).then(res => res.json());
verboseDebug('playlist=%O', playlist);
return playlist; return playlist;
} }
exports.getSpotifyPlaylist = getSpotifyPlaylist; exports.getSpotifyPlaylist = getSpotifyPlaylist;
@ -90,7 +92,7 @@ async function searchOnMyFreeMp3(query){
}) })
.then(res => res.text()) .then(res => res.text())
.then(page => { .then(page => {
debug('page=%O', page); verboseDebug('page=%O', page);
return page; return page;
}) })
.then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response })) .then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response }))
@ -106,51 +108,61 @@ function matchScore(spotifyMetas, vkmusicMetas){
const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length)); const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length));
const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length)); const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length));
const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0 const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0
debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore); verboseDebug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore);
return matchArtistScore + matchTitleScore + matchDurationScore; return matchArtistScore + matchTitleScore + matchDurationScore;
} }
exports.matchScore = matchScore; exports.matchScore = matchScore;
function generateM3U8Playlist(filesWithMetas){ function generateM3U8Playlist(playlist){
const m3uWriter = m3u.extendedWriter(); const m3uWriter = m3u.extendedWriter();
filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`)); m3uWriter.comment(`VK-MUSIC-DL SPOTIFYURL ${playlist.external_urls.spotify}`);
playlist.tracks.items.forEach(item => {
if(item.destPath){
m3uWriter.file(item.destPath, item.vkBestMatch.duration, `${item.vkBestMatch.artist} - ${item.vkBestMatch.title}`);
} else {
const artistNames = _.map(item.track.artists, 'name').join(', ');
m3uWriter.comment(`VK-MUSIC-DL Missing song ${item.track.name} - ${artistNames}`);
}
});
return m3uWriter.toString(); return m3uWriter.toString();
} }
exports.generateM3U8Playlist = generateM3U8Playlist; exports.generateM3U8Playlist = generateM3U8Playlist;
exports.main = async function main(playlistUrl){ exports.main = async function main(playlistUrl){
const playlist = await getSpotifyPlaylist(playlistUrl); const playlist = await getSpotifyPlaylist(playlistUrl);
await Promise.resolve(fs.mkdir(playlist.name)).catch({code: 'EEXIST'}, () => {}); const playlistPath = playlist.name.replace(/\/|\\/g, '');
const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => { await Promise.resolve(fs.mkdir(playlistPath)).catch({code: 'EEXIST'}, () => {});
const artistNames = _.map(track.artists, 'name').join(', '); playlist.tracks.items = await Promise.map(playlist.tracks.items, async (item) => {
debug('%s - %s', track.name, artistNames); const artistNames = _.map(item.track.artists, 'name').join(', ');
const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`); debug('%s - %s', item.track.name, artistNames);
debug('items=%O', items); const myFreeMp3Results = await searchOnMyFreeMp3(`${artistNames} ${item.track.name}`);
const bestMatch = _.chain(items).map((item) => { verboseDebug('myFreeMp3Results=%O', myFreeMp3Results);
item.score = matchScore(track, item); const bestMatch = _.chain(myFreeMp3Results).map((result) => {
return item; result.score = matchScore(item.track, result);
}).filter(item => item.score && item.url).sortBy('score').last().value(); return result;
if(!bestMatch){ }).filter(result => result.score && result.url).sortBy('score').last().value();
console.log(`You are on your own for ${track.name} - ${artistNames}`); if(bestMatch){
return; verboseDebug('bestMatch=%O', bestMatch);
} item.vkBestMatch = bestMatch;
// TODO handle '/' in title because path.join doesn't escape them const destPath = `${bestMatch.artist} - ${bestMatch.title}.mp3`.replace(/\/|\\/g, '');
bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`;
debug('bestMatch=%O', bestMatch);
try { try {
await fs.access(path.join(playlist.name, bestMatch.path)).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems await fs.access(path.join(playlistPath, destPath)).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems
await fetch(bestMatch.url).then(async res => { await fetch(bestMatch.url).then(async res => {
await promisepipe(res.body, createWriteStream(path.join(playlist.name, bestMatch.path))); // TODO check the mime type here
await promisepipe(res.body, createWriteStream(path.join(playlistPath, destPath)));
}); });
debug('Done downloading'); debug('Done downloading');
}); });
item.destPath = destPath;
} catch (err) { } catch (err) {
console.error('Download failed for ', bestMatch); console.error('Download failed for ', bestMatch);
console.error(err); console.error(err);
return null;
} }
return bestMatch; } else {
console.log(`You are on your own for ${item.track.name} - ${artistNames}`);
}
return item;
}, {concurrency: 1}); }, {concurrency: 1});
await fs.writeFile(path.join(playlist.name, `playlist.m3u8`), generateM3U8Playlist(_.compact(vkPlaylist))); await fs.writeFile(path.join(playlistPath, `playlist.m3u8`), generateM3U8Playlist(playlist));
} }