From 9e9f280f15482c82c2bcc1b167342afdde66d97c Mon Sep 17 00:00:00 2001 From: HugoPoi Date: Tue, 27 Oct 2020 23:33:10 +0100 Subject: [PATCH] feat: m3u8 have some metadata about missing songs and spotify playlist id --- README.md | 7 +++-- lib/index.js | 80 ++++++++++++++++++++++++++++++---------------------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 4409d76..938db3a 100644 --- a/README.md +++ b/README.md @@ -17,17 +17,18 @@ Inspired by [DatMusic](https://github.com/alashow/datmusic-api), [MyFreeMp3](htt *TODO* * ~~Dowload all songs in a folder~~ -* Add Spotify metadatas in the m3u8 file +* ~~Add Spotify metadatas in the m3u8 file~~ * Add proper command line options * Add CLI progress info -* Having placeholder for files that can't be downloaded, maybe as - comment in the m3u8 file +* ~~Having placeholder for files that can't be downloaded, maybe as + comment in the m3u8 file~~ * Having a option to retry/resync download mp3 based on the m3u8 file * Check the mime type of receive files because some are html files instead of mp3 WTF !! * Rewrite this as a plugin for Ampache * Rewrite this as a plugin for Funkwhale * Add MusicBrain ID3Tag on file +* Have a fallback for downloading song with youtube-dl :-P ## Goals diff --git a/lib/index.js b/lib/index.js index a28a6b4..60775e7 100644 --- a/lib/index.js +++ b/lib/index.js @@ -2,6 +2,7 @@ const Cheerio = require('cheerio'); const { promises: fs, createWriteStream } = require('fs'); const debug = require('debug')('vk-music-dl'); +const verboseDebug = require('debug')('vk-music-dl:debug'); const vm = require('vm'); const Promise = require('bluebird'); const _ = require('lodash'); @@ -29,7 +30,7 @@ async function getSpotifyPlaylist(playListUrl){ $('script#config').each(function(i, elem){ config = JSON.parse($(this).html()); }); - debug('config=%O', config); + verboseDebug('config=%O', config); const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1]; @@ -40,6 +41,7 @@ async function getSpotifyPlaylist(playListUrl){ } }).then(res => res.json()); + verboseDebug('playlist=%O', playlist); return playlist; } exports.getSpotifyPlaylist = getSpotifyPlaylist; @@ -90,7 +92,7 @@ async function searchOnMyFreeMp3(query){ }) .then(res => res.text()) .then(page => { - debug('page=%O', page); + verboseDebug('page=%O', page); return page; }) .then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response })) @@ -106,51 +108,61 @@ function matchScore(spotifyMetas, vkmusicMetas){ const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length)); const matchTitleScore = 1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length)); const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0 - debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore); + verboseDebug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore); return matchArtistScore + matchTitleScore + matchDurationScore; } exports.matchScore = matchScore; -function generateM3U8Playlist(filesWithMetas){ +function generateM3U8Playlist(playlist){ const m3uWriter = m3u.extendedWriter(); - filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`)); + m3uWriter.comment(`VK-MUSIC-DL SPOTIFYURL ${playlist.external_urls.spotify}`); + playlist.tracks.items.forEach(item => { + if(item.destPath){ + m3uWriter.file(item.destPath, item.vkBestMatch.duration, `${item.vkBestMatch.artist} - ${item.vkBestMatch.title}`); + } else { + const artistNames = _.map(item.track.artists, 'name').join(', '); + m3uWriter.comment(`VK-MUSIC-DL Missing song ${item.track.name} - ${artistNames}`); + } + }); return m3uWriter.toString(); } exports.generateM3U8Playlist = generateM3U8Playlist; exports.main = async function main(playlistUrl){ const playlist = await getSpotifyPlaylist(playlistUrl); - await Promise.resolve(fs.mkdir(playlist.name)).catch({code: 'EEXIST'}, () => {}); - const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => { - const artistNames = _.map(track.artists, 'name').join(', '); - debug('%s - %s', track.name, artistNames); - const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`); - debug('items=%O', items); - const bestMatch = _.chain(items).map((item) => { - item.score = matchScore(track, item); - return item; - }).filter(item => item.score && item.url).sortBy('score').last().value(); - if(!bestMatch){ - console.log(`You are on your own for ${track.name} - ${artistNames}`); - return; - } - // TODO handle '/' in title because path.join doesn't escape them - bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`; - debug('bestMatch=%O', bestMatch); - try { - await fs.access(path.join(playlist.name, bestMatch.path)).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems - await fetch(bestMatch.url).then(async res => { - await promisepipe(res.body, createWriteStream(path.join(playlist.name, bestMatch.path))); + const playlistPath = playlist.name.replace(/\/|\\/g, ''); + await Promise.resolve(fs.mkdir(playlistPath)).catch({code: 'EEXIST'}, () => {}); + playlist.tracks.items = await Promise.map(playlist.tracks.items, async (item) => { + const artistNames = _.map(item.track.artists, 'name').join(', '); + debug('%s - %s', item.track.name, artistNames); + const myFreeMp3Results = await searchOnMyFreeMp3(`${artistNames} ${item.track.name}`); + verboseDebug('myFreeMp3Results=%O', myFreeMp3Results); + const bestMatch = _.chain(myFreeMp3Results).map((result) => { + result.score = matchScore(item.track, result); + return result; + }).filter(result => result.score && result.url).sortBy('score').last().value(); + if(bestMatch){ + verboseDebug('bestMatch=%O', bestMatch); + item.vkBestMatch = bestMatch; + const destPath = `${bestMatch.artist} - ${bestMatch.title}.mp3`.replace(/\/|\\/g, ''); + try { + await fs.access(path.join(playlistPath, destPath)).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems + await fetch(bestMatch.url).then(async res => { + // TODO check the mime type here + await promisepipe(res.body, createWriteStream(path.join(playlistPath, destPath))); + }); + debug('Done downloading'); }); - debug('Done downloading'); - }); - } catch (err) { - console.error('Download failed for ', bestMatch); - console.error(err); - return null; + item.destPath = destPath; + } catch (err) { + console.error('Download failed for ', bestMatch); + console.error(err); + } + } else { + console.log(`You are on your own for ${item.track.name} - ${artistNames}`); } - return bestMatch; + return item; }, {concurrency: 1}); - await fs.writeFile(path.join(playlist.name, `playlist.m3u8`), generateM3U8Playlist(_.compact(vkPlaylist))); + await fs.writeFile(path.join(playlistPath, `playlist.m3u8`), generateM3U8Playlist(playlist)); }