vk-music-dl/lib/index.js

'use strict';
const Cheerio = require('cheerio');
const { promises: fs, createWriteStream } = require('fs');
const debug = require('debug')('vk-music-dl');
const verboseDebug = require('debug')('vk-music-dl:debug');
const vm = require('vm');
const Promise = require('bluebird');
const _ = require('lodash');
const leven = require('leven');
const { URL, URLSearchParams } = require('url');
const fetch = require('node-fetch');
const m3u = require('m3u');
const promisepipe = require('promisepipe');
const path = require('path');
require('dotenv').config();

const userAgentString = 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0';

async function getSpotifyPlaylist(playListUrl){
  const spotifyPlaylistPageContent = await fetch(playListUrl, {
    headers: {
      'User-Agent': userAgentString,
    }
  })
    .then(res => res.text())

  const $ = Cheerio.load(spotifyPlaylistPageContent);

  let config;
  $('script#config').each(function(i, elem){
    config = JSON.parse($(this).html());
  });
  verboseDebug('config=%O', config);

  const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/.exec(playListUrl)[1];

  const playlist = await fetch(`https://api.spotify.com/v1/playlists/${playlistSpotifyId}?type=track%2Cepisode`, {
    headers: {
      'User-Agent': userAgentString,
      Authorization: `Bearer ${config.accessToken}`
    }
  }).then(res => res.json());

  verboseDebug('playlist=%O', playlist);
  return playlist;
}
exports.getSpotifyPlaylist = getSpotifyPlaylist;

async function searchOnVkMusic(query){
  const url = new URL('https://api.vk.com/method/audio.search');
  url.search = new URLSearchParams({
    v: '5.71',
    access_token: process.env.ACCESS_TOKEN,
    q: query,
    count: 200,
  }).toString();

  return await fetch(url, {
    headers: {
      'User-Agent': 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)',
    }
  })
  .then(res => res.json())
  .then(responseBody => {
    if(responseBody.error){
      throw _.assign(responseBody.error, new Error(responseBody.error.error_msg));
    } else {
      return responseBody.response.items;
    }
  });
}
exports.searchOnVkMusic = searchOnVkMusic;

async function searchOnMyFreeMp3(query){
  const url = new URL('https://myfreemp3.vip/api/search.php?callback=jQuery666');

  return await fetch(url, {
    method: 'POST',
    headers: {
      'User-Agent': userAgentString,
      'X-Requested-With': 'XMLHttpRequest',
      Accept: 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
      'Accept-Language': 'fr-FR,fr;q=0.5',
      Referer: 'https://myfreemp3.vip/',
      Origin: 'https://myfreemp3.vip',
      //Cookie: '__cfduid=d1821f08faee82ed9d4bf93b1b5fb9d3e1601196911; musicLang=en'
    },
    body: new URLSearchParams({
      q: query,
      page: 0,
    }),
  })
  .then(res => res.text())
  .then(page => {
    verboseDebug('page=%O', page);
    return page;
  })
  .then(jsonp => vm.runInNewContext(jsonp, { jQuery666: (payload) => payload.response }))
  .then(items => _.filter(items, item => !_.isString(item)));
}
exports.searchOnMyFreeMp3 = searchOnMyFreeMp3;

function matchScore(spotifyMetas, vkmusicMetas){
  const originalArtistNames = _.map(spotifyMetas.artists, 'name').join(', ').toLowerCase();
  const originalTitle = spotifyMetas.name.toLowerCase();
  const originalDuration = Math.round(spotifyMetas.duration_ms/1000);

  const matchArtistScore = 1 - (leven(originalArtistNames, _.get(vkmusicMetas, 'artist', '').toLowerCase()) / Math.max(originalArtistNames.length, _.get(vkmusicMetas, 'artist', '').length));
  const matchTitleScore =  1 - (leven(originalTitle, _.get(vkmusicMetas, 'title', '').toLowerCase()) / Math.max(originalTitle.length, _.get(vkmusicMetas, 'title', '').length));
  const matchDurationScore = 1 - (Math.abs(originalDuration - _.get(vkmusicMetas, 'duration', 0)) / originalDuration); // TODO this can return more than 1 or less than 0
  verboseDebug('matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f', matchArtistScore, matchTitleScore, matchDurationScore);
  return matchArtistScore + matchTitleScore + matchDurationScore;
}
exports.matchScore = matchScore;

function generateM3U8Playlist(playlist){
  const m3uWriter = m3u.extendedWriter();
  m3uWriter.comment(`VK-MUSIC-DL SPOTIFYURL ${playlist.external_urls.spotify}`);
  playlist.tracks.items.forEach(item => {
    if(item.destPath){
      m3uWriter.file(item.destPath, item.vkBestMatch.duration, `${item.vkBestMatch.artist} - ${item.vkBestMatch.title}`);
    } else {
      const artistNames = _.map(item.track.artists, 'name').join(', ');
      m3uWriter.comment(`VK-MUSIC-DL Missing song ${item.track.name} - ${artistNames}`);
    }
  });
  return m3uWriter.toString();
}
exports.generateM3U8Playlist = generateM3U8Playlist;

exports.main = async function main(playlistUrl){
  const playlist = await getSpotifyPlaylist(playlistUrl);
  const playlistPath = playlist.name.replace(/\/|\\/g, '');
  await Promise.resolve(fs.mkdir(playlistPath)).catch({code: 'EEXIST'}, () => {});
  playlist.tracks.items = await Promise.map(playlist.tracks.items, async (item) => {
    const artistNames = _.map(item.track.artists, 'name').join(', ');
    debug('%s - %s', item.track.name, artistNames);
    const myFreeMp3Results = await searchOnMyFreeMp3(`${artistNames} ${item.track.name}`);
    verboseDebug('myFreeMp3Results=%O', myFreeMp3Results);
    const bestMatch = _.chain(myFreeMp3Results).map((result) => {
      result.score = matchScore(item.track, result);
      return result;
    }).filter(result => result.score && result.url).sortBy('score').last().value();
    if(bestMatch){
      verboseDebug('bestMatch=%O', bestMatch);
      item.vkBestMatch = bestMatch;
      const destPath = `${bestMatch.artist} - ${bestMatch.title}.mp3`.replace(/\/|\\/g, '');
      try {
        await fs.access(path.join(playlistPath, destPath)).catch(async () => { // TODO find a proper way to not re-download, lower/uppercase problems
          await fetch(bestMatch.url).then(async res => {
            // TODO check the mime type here
            await promisepipe(res.body, createWriteStream(path.join(playlistPath, destPath)));
          });
          debug('Done downloading');
        });
        item.destPath = destPath;
      } catch (err) {
        console.error('Download failed for ', bestMatch);
        console.error(err);
      }
    } else {
      console.log(`You are on your own for ${item.track.name} - ${artistNames}`);
    }
    return item;
  }, {concurrency: 1});

  await fs.writeFile(path.join(playlistPath, `playlist.m3u8`), generateM3U8Playlist(playlist));
}