Merge pull request 'feat: add-test and some fixes' (#1) from feat/add-test into master

Reviewed-on: #1

README.md

@@ -17,6 +17,7 @@ Inspired by [DatMusic](https://github.com/alashow/datmusic-api), [MyFreeMp3](htt
 *TODO*
 
 * Dowload all songs in a folder
+* Add Spotify metadatas in the m3u8 file
 * Add proper command line options
 * Rewrite this as a plugin for Ampache
 * Rewrite this as a plugin for Funkwhale
@@ -32,3 +33,11 @@ Inspired by [DatMusic](https://github.com/alashow/datmusic-api), [MyFreeMp3](htt
 
 1. Install nodejs/npm
 1. `npx vk-music-dl https://open.spotify.com/playlist/6LgeEhc97Azxq6sinJQt6w`
+
+## Usage as library
+
+```javascript
+const {main} = require('vk-music-dl/lib');
+
+main('https://open.spotify.com/playlist/6LgeEhc97Azxq6sinJQt6w');
+```

index.js

@@ -1,148 +1,6 @@
 #!/usr/bin/env node
 'use strict';
-const Cheerio = require('cheerio');
-const { promises: fs, createWriteStream } = require('fs');
-const debug = require('debug')('vk-music-dl');
-const vm = require('vm');
-const Promise = require('bluebird');
-const _ = require('lodash');
-const leven = require('leven');
-const { URL, URLSearchParams } = require('url');
-const fetch = require('node-fetch');
-const m3u = require('m3u');
-const promisepipe = require('promisepipe');
 require('dotenv').config();
-
-
-async function getSpotifyPlaylist(playListUrl){
-  const spotifyPlaylistPageContent = await fetch(playListUrl, {
-    headers: {
-      'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
-    }
-  })
-    .then(res => res.text())
-
-  const $ = Cheerio.load(spotifyPlaylistPageContent);
-
-  let config;
-  $('script#config').each(function(i, elem){
-    config = JSON.parse($(this).html());
-  });
-  debug('config=%O', config);
-
-  const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1];
-
-  const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, {
-    headers: {
-      'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
-      Authorization: `Bearer ${config.accessToken}`
-    }
-  }).then(res => res.json());
-
-  return playlist;
-}
-
-async function searchOnVkMusic(query){
-  const url = new URL('https://api.vk.com/method/audio.search');
-  url.search = new URLSearchParams({
-    v: '5.71',
-    access_token: process.env.ACCESS_TOKEN,
-    q: query,
-    count: 200,
-  }).toString();
-
-  return await fetch(url, {
-    headers: {
-      'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)',
-    }
-  })
-  .then(res => res.json())
-  .then(responseBody => {
-    if(responseBody.error){
-      throw _.assign(responseBody.error, new Error(responseBody.error.error_msg));
-    } else {
-      return responseBody.response.items;
-    }
-  });
-}
-
-async function searchOnMyFreeMp3(query){
-  const url = new URL('https://myfreemp3cc.com/api/search.php?callback=jQuery666');
-
-  return await fetch(url, {
-    method: 'POST',
-    body: new URLSearchParams({
-      q: query,
-      page: 0,
-    }),
-  })
-  .then(res => res.text())
-  .then(page => {
-    debug('page=%O', page);
-    return page;
-  })
-  .then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response }))
-  .then(items => _.filter(items, item => !_.isString(item)));
-}
-
-function matchScore(spotifyMetas, vkmusicMetas){
-  const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase();
-  const originalTitle = spotifyMetas.name.toLowerCase();
-  const originalDuration = Math.round(spotifyMetas.duration_ms/1000);
-
-  const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length));
-  const matchTitleScore =  1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length));
-  const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0
-  debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore);
-  return matchArtistScore + matchTitleScore + matchDurationScore;
-}
-
-async function generateM3U8Playlist(filesWithMetas){
-  const m3uWriter = m3u.extendedWriter();
-  filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`));
-  await fs.writeFile(`playlist.m3u8`, m3uWriter.toString());
-}
-
-async function main(playlistUrl){
-  const playlist = await getSpotifyPlaylist(playlistUrl);
-  const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => {
-    const artistNames = _.map(track.artists, 'name').join(', ');
-    debug('%s - %s', track.name, artistNames);
-    const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`);
-    debug('items=%O', items);
-    const bestMatch = _.chain(items).map((item) => {
-      item.score = matchScore(track, item);
-      return item;
-    }).filter(item => item.score && item.url).sortBy('score').last().value();
-    if(!bestMatch){
-      console.log(`You are on your own for ${track.name} - ${artistNames}`);
-      return;
-    }
-    bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`;
-    debug('bestMatch=%O', bestMatch);
-    try {
-      await fs.access(bestMatch.path).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems
-        await fetch(bestMatch.url).then(async res => {
-          await promisepipe(res.body, createWriteStream(bestMatch.path));
-        });
-        debug('Done downloading');
-      });
-    } catch (err) {
-      console.error('Download failed for ', bestMatch);
-      console.error(err);
-      return null;
-    }
-    return bestMatch;
-  }, {concurrency: 1});
-
-  await generateM3U8Playlist(_.compact(vkPlaylist));
-}
+const { main } = require('./lib');
 
 main(process.argv[2]).catch(err => console.error(err));
-
-async function test(query){
-  const response = await searchOnMyFreeMp3(query);
-  console.log(response);
-}
-
-//test(process.argv[2]);

lib/index.js

@@ -0,0 +1,155 @@
+'use strict';
+const Cheerio = require('cheerio');
+const { promises: fs, createWriteStream } = require('fs');
+const debug = require('debug')('vk-music-dl');
+const vm = require('vm');
+const Promise = require('bluebird');
+const _ = require('lodash');
+const leven = require('leven');
+const { URL, URLSearchParams } = require('url');
+const fetch = require('node-fetch');
+const m3u = require('m3u');
+const promisepipe = require('promisepipe');
+const path = require('path');
+require('dotenv').config();
+
+const userAgentString = 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0';
+
+async function getSpotifyPlaylist(playListUrl){
+  const spotifyPlaylistPageContent = await fetch(playListUrl, {
+    headers: {
+      'User-Agent': userAgentString,
+    }
+  })
+    .then(res => res.text())
+
+  const $ = Cheerio.load(spotifyPlaylistPageContent);
+
+  let config;
+  $('script#config').each(function(i, elem){
+    config = JSON.parse($(this).html());
+  });
+  debug('config=%O', config);
+
+  const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1];
+
+  const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, {
+    headers: {
+      'User-Agent': userAgentString,
+      Authorization: `Bearer ${config.accessToken}`
+    }
+  }).then(res => res.json());
+
+  return playlist;
+}
+exports.getSpotifyPlaylist = getSpotifyPlaylist;
+
+async function searchOnVkMusic(query){
+  const url = new URL('https://api.vk.com/method/audio.search');
+  url.search = new URLSearchParams({
+    v: '5.71',
+    access_token: process.env.ACCESS_TOKEN,
+    q: query,
+    count: 200,
+  }).toString();
+
+  return await fetch(url, {
+    headers: {
+      'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)',
+    }
+  })
+  .then(res => res.json())
+  .then(responseBody => {
+    if(responseBody.error){
+      throw _.assign(responseBody.error, new Error(responseBody.error.error_msg));
+    } else {
+      return responseBody.response.items;
+    }
+  });
+}
+exports.searchOnVkMusic = searchOnVkMusic;
+
+async function searchOnMyFreeMp3(query){
+  const url = new URL('https://myfreemp3.vip/api/search.php?callback=jQuery666');
+
+  return await fetch(url, {
+    method: 'POST',
+    headers: {
+      'User-Agent': userAgentString,
+      'X-Requested-With': 'XMLHttpRequest',
+      Accept: 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
+      'Accept-Language': 'fr-FR,fr;q=0.5',
+      Referer: 'https://myfreemp3.vip/',
+      Origin: 'https://myfreemp3.vip',
+      //Cookie: '__cfduid=d1821f08faee82ed9d4bf93b1b5fb9d3e1601196911; musicLang=en'
+    },
+    body: new URLSearchParams({
+      q: query,
+      page: 0,
+    }),
+  })
+  .then(res => res.text())
+  .then(page => {
+    debug('page=%O', page);
+    return page;
+  })
+  .then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response }))
+  .then(items => _.filter(items, item => !_.isString(item)));
+}
+exports.searchOnMyFreeMp3 = searchOnMyFreeMp3;
+
+function matchScore(spotifyMetas, vkmusicMetas){
+  const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase();
+  const originalTitle = spotifyMetas.name.toLowerCase();
+  const originalDuration = Math.round(spotifyMetas.duration_ms/1000);
+
+  const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length));
+  const matchTitleScore =  1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length));
+  const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0
+  debug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore);
+  return matchArtistScore + matchTitleScore + matchDurationScore;
+}
+exports.matchScore = matchScore;
+
+function generateM3U8Playlist(filesWithMetas){
+  const m3uWriter = m3u.extendedWriter();
+  filesWithMetas.forEach(fileWithMetas => m3uWriter.file(fileWithMetas.path, fileWithMetas.duration, `${fileWithMetas.artist} - ${fileWithMetas.title}`));
+  return m3uWriter.toString();
+}
+exports.generateM3U8Playlist = generateM3U8Playlist;
+
+exports.main = async function main(playlistUrl){
+  const playlist = await getSpotifyPlaylist(playlistUrl);
+  await Promise.resolve(fs.mkdir(playlist.name)).catch({code: 'EEXIST'}, () => {});
+  const vkPlaylist = await Promise.map(playlist.tracks.items, async ({track}) => {
+    const artistNames = _.map(track.artists, 'name').join(', ');
+    debug('%s - %s', track.name, artistNames);
+    const items = await searchOnMyFreeMp3(`${artistNames} ${track.name}`);
+    debug('items=%O', items);
+    const bestMatch = _.chain(items).map((item) => {
+      item.score = matchScore(track, item);
+      return item;
+    }).filter(item => item.score && item.url).sortBy('score').last().value();
+    if(!bestMatch){
+      console.log(`You are on your own for ${track.name} - ${artistNames}`);
+      return;
+    }
+    bestMatch.path = `${bestMatch.artist} - ${bestMatch.title}.mp3`;
+    debug('bestMatch=%O', bestMatch);
+    try {
+      await fs.access(path.join(playlist.name, bestMatch.path)).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems
+        await fetch(bestMatch.url).then(async res => {
+          await promisepipe(res.body, createWriteStream(path.join(playlist.name, bestMatch.path)));
+        });
+        debug('Done downloading');
+      });
+    } catch (err) {
+      console.error('Download failed for ', bestMatch);
+      console.error(err);
+      return null;
+    }
+    return bestMatch;
+  }, {concurrency: 1});
+
+  await fs.writeFile(path.join(playlist.name, `playlist.m3u8`), generateM3U8Playlist(_.compact(vkPlaylist)));
+}

package.json

@@ -4,7 +4,7 @@
   "description": "Download Spotify playlist as mp3 from VKontakte",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "mocha"
   },
   "bin": {
     "vk-music-dl": "./index.js"
@@ -21,6 +21,7 @@
     "downloader"
   ],
   "files": [
+    "lib",
     "index.js",
     "README.md"
   ],
@@ -36,5 +37,7 @@
     "node-fetch": "^2.6.0",
     "promisepipe": "^3.0.0"
   },
-  "devDependencies": {}
+  "devDependencies": {
+    "mocha": "^8.1.3"
+  }
 }

test/lib.test.js

@@ -0,0 +1,14 @@
+'use strict';
+const assert = require('assert');
+const {getSpotifyPlaylist} = require('../lib');
+
+describe('getSpotifyPlaylist', function(){
+
+  it('acquire and parse playlist from Spotify', async function(){
+    const playlist = await getSpotifyPlaylist('https://open.spotify.com/playlist/27OFA3NDwoLWtmdMvowA3S');
+    assert.strictEqual(playlist.name, 'The Best Yoga & Cool-Down Songs');
+    assert.strictEqual(playlist.type, 'playlist');
+    assert.strictEqual(playlist.tracks.items.length, 10);
+  });
+
+});