2020-09-27 10:39:34 +02:00
'use strict' ;
const Cheerio = require ( 'cheerio' ) ;
const { promises : fs , createWriteStream } = require ( 'fs' ) ;
const debug = require ( 'debug' ) ( 'vk-music-dl' ) ;
2020-10-27 23:33:10 +01:00
const verboseDebug = require ( 'debug' ) ( 'vk-music-dl:debug' ) ;
2020-09-27 10:39:34 +02:00
const vm = require ( 'vm' ) ;
const Promise = require ( 'bluebird' ) ;
const _ = require ( 'lodash' ) ;
const leven = require ( 'leven' ) ;
const { URL , URLSearchParams } = require ( 'url' ) ;
const fetch = require ( 'node-fetch' ) ;
const m3u = require ( 'm3u' ) ;
const promisepipe = require ( 'promisepipe' ) ;
2020-09-27 11:22:10 +02:00
const path = require ( 'path' ) ;
2020-09-27 10:39:34 +02:00
require ( 'dotenv' ) . config ( ) ;
2020-09-27 11:22:10 +02:00
const userAgentString = 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0' ;
2020-09-27 10:39:34 +02:00
async function getSpotifyPlaylist ( playListUrl ) {
const spotifyPlaylistPageContent = await fetch ( playListUrl , {
headers : {
2020-09-27 11:22:10 +02:00
'User-Agent' : userAgentString ,
2020-09-27 10:39:34 +02:00
}
} )
. then ( res => res . text ( ) )
const $ = Cheerio . load ( spotifyPlaylistPageContent ) ;
let config ;
$ ( 'script#config' ) . each ( function ( i , elem ) {
config = JSON . parse ( $ ( this ) . html ( ) ) ;
} ) ;
2020-10-27 23:33:10 +01:00
verboseDebug ( 'config=%O' , config ) ;
2020-09-27 10:39:34 +02:00
const playlistSpotifyId = /\/playlist\/([^\/]+)\/?$/ . exec ( playListUrl ) [ 1 ] ;
const playlist = await fetch ( ` https://api.spotify.com/v1/playlists/ ${ playlistSpotifyId } ?type=track%2Cepisode ` , {
headers : {
2020-09-27 11:22:10 +02:00
'User-Agent' : userAgentString ,
2020-09-27 10:39:34 +02:00
Authorization : ` Bearer ${ config . accessToken } `
}
} ) . then ( res => res . json ( ) ) ;
2020-10-27 23:33:10 +01:00
verboseDebug ( 'playlist=%O' , playlist ) ;
2020-09-27 10:39:34 +02:00
return playlist ;
}
exports . getSpotifyPlaylist = getSpotifyPlaylist ;
async function searchOnVkMusic ( query ) {
const url = new URL ( 'https://api.vk.com/method/audio.search' ) ;
url . search = new URLSearchParams ( {
v : '5.71' ,
access _token : process . env . ACCESS _TOKEN ,
q : query ,
count : 200 ,
} ) . toString ( ) ;
return await fetch ( url , {
headers : {
'User-Agent' : 'KateMobileAndroid/48.2 lite-433 (Android 8.1.0; SDK 27; arm64-v8a; Google Pixel 2 XL; en)' ,
}
} )
. then ( res => res . json ( ) )
. then ( responseBody => {
if ( responseBody . error ) {
throw _ . assign ( responseBody . error , new Error ( responseBody . error . error _msg ) ) ;
} else {
return responseBody . response . items ;
}
} ) ;
}
exports . searchOnVkMusic = searchOnVkMusic ;
async function searchOnMyFreeMp3 ( query ) {
const url = new URL ( 'https://myfreemp3.vip/api/search.php?callback=jQuery666' ) ;
return await fetch ( url , {
method : 'POST' ,
2020-09-27 11:22:10 +02:00
headers : {
'User-Agent' : userAgentString ,
'X-Requested-With' : 'XMLHttpRequest' ,
Accept : 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01' ,
'Accept-Language' : 'fr-FR,fr;q=0.5' ,
Referer : 'https://myfreemp3.vip/' ,
Origin : 'https://myfreemp3.vip' ,
//Cookie: '__cfduid=d1821f08faee82ed9d4bf93b1b5fb9d3e1601196911; musicLang=en'
} ,
2020-09-27 10:39:34 +02:00
body : new URLSearchParams ( {
q : query ,
page : 0 ,
} ) ,
} )
. then ( res => res . text ( ) )
. then ( page => {
2020-10-27 23:33:10 +01:00
verboseDebug ( 'page=%O' , page ) ;
2020-09-27 10:39:34 +02:00
return page ;
} )
. then ( jsonp => vm . runInNewContext ( jsonp , { jQuery666 : ( payload ) => payload . response } ) )
. then ( items => _ . filter ( items , item => ! _ . isString ( item ) ) ) ;
}
exports . searchOnMyFreeMp3 = searchOnMyFreeMp3 ;
function matchScore ( spotifyMetas , vkmusicMetas ) {
const originalArtistNames = _ . map ( spotifyMetas . artists , 'name' ) . join ( ', ' ) . toLowerCase ( ) ;
const originalTitle = spotifyMetas . name . toLowerCase ( ) ;
const originalDuration = Math . round ( spotifyMetas . duration _ms / 1000 ) ;
const matchArtistScore = 1 - ( leven ( originalArtistNames , _ . get ( vkmusicMetas , 'artist' , '' ) . toLowerCase ( ) ) / Math . max ( originalArtistNames . length , _ . get ( vkmusicMetas , 'artist' , '' ) . length ) ) ;
const matchTitleScore = 1 - ( leven ( originalTitle , _ . get ( vkmusicMetas , 'title' , '' ) . toLowerCase ( ) ) / Math . max ( originalTitle . length , _ . get ( vkmusicMetas , 'title' , '' ) . length ) ) ;
const matchDurationScore = 1 - ( Math . abs ( originalDuration - _ . get ( vkmusicMetas , 'duration' , 0 ) ) / originalDuration ) ; // TODO this can return more than 1 or less than 0
2020-10-27 23:33:10 +01:00
verboseDebug ( 'matchArtistScore=%f matchTitleScore=%f matchDurationScore=%f' , matchArtistScore , matchTitleScore , matchDurationScore ) ;
2020-09-27 10:39:34 +02:00
return matchArtistScore + matchTitleScore + matchDurationScore ;
}
exports . matchScore = matchScore ;
2020-10-27 23:33:10 +01:00
function generateM3U8Playlist ( playlist ) {
2020-09-27 10:39:34 +02:00
const m3uWriter = m3u . extendedWriter ( ) ;
2020-10-27 23:33:10 +01:00
m3uWriter . comment ( ` VK-MUSIC-DL SPOTIFYURL ${ playlist . external _urls . spotify } ` ) ;
playlist . tracks . items . forEach ( item => {
if ( item . destPath ) {
m3uWriter . file ( item . destPath , item . vkBestMatch . duration , ` ${ item . vkBestMatch . artist } - ${ item . vkBestMatch . title } ` ) ;
} else {
const artistNames = _ . map ( item . track . artists , 'name' ) . join ( ', ' ) ;
m3uWriter . comment ( ` VK-MUSIC-DL Missing song ${ item . track . name } - ${ artistNames } ` ) ;
}
} ) ;
2020-09-27 10:39:34 +02:00
return m3uWriter . toString ( ) ;
}
exports . generateM3U8Playlist = generateM3U8Playlist ;
exports . main = async function main ( playlistUrl ) {
const playlist = await getSpotifyPlaylist ( playlistUrl ) ;
2020-10-27 23:33:10 +01:00
const playlistPath = playlist . name . replace ( /\/|\\/g , '' ) ;
await Promise . resolve ( fs . mkdir ( playlistPath ) ) . catch ( { code : 'EEXIST' } , ( ) => { } ) ;
playlist . tracks . items = await Promise . map ( playlist . tracks . items , async ( item ) => {
const artistNames = _ . map ( item . track . artists , 'name' ) . join ( ', ' ) ;
debug ( '%s - %s' , item . track . name , artistNames ) ;
const myFreeMp3Results = await searchOnMyFreeMp3 ( ` ${ artistNames } ${ item . track . name } ` ) ;
verboseDebug ( 'myFreeMp3Results=%O' , myFreeMp3Results ) ;
const bestMatch = _ . chain ( myFreeMp3Results ) . map ( ( result ) => {
result . score = matchScore ( item . track , result ) ;
return result ;
} ) . filter ( result => result . score && result . url ) . sortBy ( 'score' ) . last ( ) . value ( ) ;
if ( bestMatch ) {
verboseDebug ( 'bestMatch=%O' , bestMatch ) ;
item . vkBestMatch = bestMatch ;
const destPath = ` ${ bestMatch . artist } - ${ bestMatch . title } .mp3 ` . replace ( /\/|\\/g , '' ) ;
try {
await fs . access ( path . join ( playlistPath , destPath ) ) . catch ( async ( ) => { // TODO find a proper way to not re-download, lower/uppercase problems
await fetch ( bestMatch . url ) . then ( async res => {
// TODO check the mime type here
await promisepipe ( res . body , createWriteStream ( path . join ( playlistPath , destPath ) ) ) ;
} ) ;
debug ( 'Done downloading' ) ;
2020-09-27 10:39:34 +02:00
} ) ;
2020-10-27 23:33:10 +01:00
item . destPath = destPath ;
} catch ( err ) {
console . error ( 'Download failed for ' , bestMatch ) ;
console . error ( err ) ;
}
} else {
console . log ( ` You are on your own for ${ item . track . name } - ${ artistNames } ` ) ;
2020-09-27 10:39:34 +02:00
}
2020-10-27 23:33:10 +01:00
return item ;
2020-09-27 10:39:34 +02:00
} , { concurrency : 1 } ) ;
2020-10-27 23:33:10 +01:00
await fs . writeFile ( path . join ( playlistPath , ` playlist.m3u8 ` ) , generateM3U8Playlist ( playlist ) ) ;
2020-09-27 10:39:34 +02:00
}