dbroqua/MusicTopus
dbroqua/MusicTopus
1
1
Fork 0
Code Issues 9 Pull requests Releases 14 Activity

issue/37 (#42)

Browse source 
Co-authored-by: dbroqua <contact@darkou.fr>
Reviewed-on: #42
This commit is contained in:
Damien Broqua 2022-04-09 00:07:22 +02:00
parent 182aa7a6a6
commit ae4b7b6de0
14 changed files with 822 additions and 472 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
README.md
View file

@ -62,7 +62,7 @@ Le site est accessible sur [http://localhost:PORT](http://localhost:PORT).
#### Standalone
Pour la version standalone je vous conseille de faire un script embarquant les variables d'environnement que vous souhaitez modifier :
Pour la version standalone je vous conseille de faire un script embarquant les variables d'environnement que vous souhaitez modifier ([voir à la fin pour la liste des variables](#env-file)) :
```bash
#! /bin/bash
@ -184,6 +184,26 @@ server {
Une fois le vhost activé (lien symbolique dans le dossier site-enable) et nginx rechargé votre site sera alors accessible en https.
### Jobs
Par défaut toute les images des albums sont affichées depuis Discogs. Cependant avec les temps les urls deviennent invalides. Pour éviter cela lors de l'ajout d'un album à votre collection un job est créé. Ce job a pour rôle de stocker les images sur un bucket s3.
Pour lancer les jobs il faut mettre en place une tâche cron qui sera éxécutée toute les heures (par exemple).
Exemple de crontab :
```crontab
0 * * * * curl 'http://localhost:3001/jobs' \
-H 'JOBS_HEADER_KEY: JOBS_HEADER_VALUE' \
-H 'Accept: application/json'
30 * * * * curl 'http://localhost:3001/jobs?state=ERROR' \
-H 'JOBS_HEADER_KEY: JOBS_HEADER_VALUE' \
-H 'Accept: application/json'
```
N'oubliez pas de remplacer `localhost:30001`, `JOBS_HEADER_KEY` et `JOBS_HEADER_VALUE` par les bonnes valeurs.
La première ligne permet de parcourir tous les nouveaux jobs alors que la seconde permet de relancer les jobs en erreurs (après 5 tentatives le job est marqué comme définitivement perdu).
### Fichier .env {#env-file}
Voici la liste des variables configurables :
@ -198,10 +218,17 @@ FORMSPREE_ID # Id du formulaire formspree pour la page "nous-contacter"
MATOMO_URL # Url vers l'instance matomo (exemple: https://analytics.darkou.fr/)
MATOMO_ID # Id du site sur votre instance matomo (exemple: 1)
SITE_NAME # Nom du site (utilisé dans le titre des pages)
AWS_ACCESS_KEY_ID # Clé d'accès AWS
AWS_SECRET_ACCESS_KEY # Clé secrète AWS
S3_ENDPOINT # Url de l'instance aws (s3.fr-par.scw.cloud pour scaleway france par exemple)
S3_SIGNATURE # Version de la signature AWS (s3v4 pour scaleway par exemple)
S3_BASEFOLDER # Nom du sous dossier dans lequel seront mis les pochettes des albums
S3_BUCKET # Nom du bucket
JOBS_HEADER_KEY # Nom du header utilisé pour l'identification des tâches cron (par exemple musictopus)
JOBS_HEADER_VALUE # Valeur de la clé
```
## Contributeurs
- Damien Broqua (développeur principal du projet)
- Brunus (Logo et fournisseur d'idées :wink: )

8
docker-compose.yml.dev
View file

@ -28,6 +28,14 @@ services:
MATOMO_URL: ${MATOMO_URL}
MATOMO_ID: ${MATOMO_ID}
SITE_NAME: ${SITE_NAME}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
S3_BASEFOLDER: ${S3_BASEFOLDER}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_SIGNATURE: ${S3_SIGNATURE}
JOBS_HEADER_KEY: ${JOBS_HEADER_KEY}
JOBS_HEADER_VALUE: ${JOBS_HEADER_VALUE}
networks:
- musictopus
musictopus-db:

8
docker-compose.yml.prod
View file

@ -28,6 +28,14 @@ services:
MATOMO_URL: ${MATOMO_URL}
MATOMO_ID: ${MATOMO_ID}
SITE_NAME: ${SITE_NAME}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
S3_BASEFOLDER: ${S3_BASEFOLDER}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_SIGNATURE: ${S3_SIGNATURE}
JOBS_HEADER_KEY: ${JOBS_HEADER_KEY}
JOBS_HEADER_VALUE: ${JOBS_HEADER_VALUE}
networks:
- musictopus
musictopus-db:

3
package.json
View file

@ -41,6 +41,7 @@
"@babel/cli": "^7.17.0",
"@babel/core": "^7.17.2",
"@babel/preset-env": "^7.16.11",
"aws-sdk": "^2.1110.0",
"axios": "^0.26.0",
"connect-ensure-login": "^0.1.1",
"connect-flash": "^0.1.1",
@ -60,10 +61,12 @@
"mongoose-unique-validator": "^3.0.0",
"npm-run-all": "^4.1.5",
"passport": "^0.5.2",
"passport-custom": "^1.1.1",
"passport-http": "^0.3.0",
"passport-local": "^1.0.0",
"rimraf": "^3.0.2",
"sass": "^1.49.7",
"uuid": "^8.3.2",
"vue": "^3.2.31"
},
"nodemonConfig": {

15
src/app.js
View file

@ -7,6 +7,8 @@ import flash from "connect-flash";
import session from "express-session";
import MongoStore from "connect-mongo";
import passportConfig from "./libs/passport";
import config, { env, mongoDbUri, secret } from "./config";
import { isXhr } from "./helpers";
@ -15,15 +17,13 @@ import indexRouter from "./routes";
import maCollectionRouter from "./routes/ma-collection";
import collectionRouter from "./routes/collection";
import importJobsRouter from "./routes/jobs";
import importAlbumRouterApiV1 from "./routes/api/v1/albums";
import importSearchRouterApiV1 from "./routes/api/v1/search";
import importMeRouterApiV1 from "./routes/api/v1/me";
// Mongoose schema init
require("./models/users");
require("./models/albums");
require("./libs/passport")(passport);
passportConfig(passport);
mongoose
.connect(mongoDbUri, { useNewUrlParser: true, useUnifiedTopology: true })
@ -46,10 +46,10 @@ const sess = {
const app = express();
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(flash());
app.use(express.json({ limit: "50mb" }));
app.use(express.urlencoded({ extended: false, limit: "50mb" }));
app.use(session(sess));
@ -85,6 +85,7 @@ app.use(
app.use("/", indexRouter);
app.use("/ma-collection", maCollectionRouter);
app.use("/collection", collectionRouter);
app.use("/jobs", importJobsRouter);
app.use("/api/v1/albums", importAlbumRouterApiV1);
app.use("/api/v1/search", importSearchRouterApiV1);
app.use("/api/v1/me", importMeRouterApiV1);

9
src/config/index.js
View file

@ -8,4 +8,13 @@ module.exports = {
matomoUrl: process.env.MATOMO_URL || "",
matomoId: process.env.MATOMO_ID || "",
siteName: process.env.SITE_NAME || "MusicTopus",
awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID,
awsSecretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
s3BaseFolder: process.env.S3_BASEFOLDER || "dev",
s3Bucket: process.env.S3_BUCKET || "musictopus",
s3Endpoint: process.env.S3_ENDPOINT || "s3.fr-par.scw.cloud",
s3Signature: process.env.S3_SIGNATURE || "s3v4",
jobsHeaderKey: process.env.JOBS_HEADER_KEY || "musictopus",
jobsHeaderValue:
process.env.JOBS_HEADER_VALUE || "ooYee9xok7eigo2shiePohyoGh1eepew",
};

72
src/libs/aws.js Normal file
View file

@ -0,0 +1,72 @@
import AWS from "aws-sdk";
import fs from "fs";
import path from "path";
import axios from "axios";
import { v4 as uuid } from "uuid";
import {
awsAccessKeyId,
awsSecretAccessKey,
s3BaseFolder,
s3Endpoint,
s3Bucket,
s3Signature,
} from "../config";
AWS.config.update({
accessKeyId: awsAccessKeyId,
secretAccessKey: awsSecretAccessKey,
});
/**
* Fonction permettant de stocker un fichier local sur S3
* @param {String} filename
* @param {String} file
* @param {Boolean} deleteFile
*
* @return {String}
*/
export const uploadFromFile = async (filename, file, deleteFile = false) => {
const data = await fs.readFileSync(file);
const base64data = Buffer.from(data, "binary");
const dest = path.join(s3BaseFolder, filename);
const s3 = new AWS.S3({
endpoint: s3Endpoint,
signatureVersion: s3Signature,
});
await s3
.putObject({
Bucket: s3Bucket,
Key: dest,
Body: base64data,
ACL: "public-read",
})
.promise();
if (deleteFile) {
fs.unlinkSync(file);
}
return `https://${s3Bucket}.${s3Endpoint}/${dest}`;
};
/**
* Fonction permettant de stocker un fichier provenant d'une URL sur S3
* @param {String} url
*
* @return {String}
*/
export const uploadFromUrl = async (url) => {
const filename = `${uuid()}.jpg`;
const file = `/tmp/${filename}`;
const { data } = await axios.get(url, { responseType: "arraybuffer" });
fs.writeFileSync(file, data);
return uploadFromFile(filename, file, true);
// return s3Object;
};

25
src/libs/passport.js
View file

@ -1,11 +1,13 @@
/* eslint-disable func-names */
const mongoose = require("mongoose");
const LocalStrategy = require("passport-local").Strategy;
const { BasicStrategy } = require("passport-http");
import { Strategy as LocalStrategy } from "passport-local";
import { BasicStrategy } from "passport-http";
import { Strategy as CustomStrategy } from "passport-custom";
const Users = mongoose.model("Users");
import Users from "../models/users";
module.exports = function (passport) {
import { jobsHeaderKey, jobsHeaderValue } from "../config";
export default (passport) => {
passport.serializeUser((user, done) => {
done(null, user);
});
@ -55,4 +57,17 @@ module.exports = function (passport) {
.catch(done);
})
);
passport.use(
"jobs",
new CustomStrategy((req, next) => {
const apiKey = req.headers[jobsHeaderKey];
if (apiKey === jobsHeaderValue) {
return next(null, {
username: "jobs",
});
}
return next(null, false, "Oops! Identifiants incorrects");
})
);
};

477
src/middleware/Albums.js
View file

@ -1,468 +1,18 @@
import moment from "moment";
import momenttz from "moment-timezone";
import xl from "excel4node";
import Pages from "./Pages";
import Export from "./Export";
import AlbumsModel from "../models/albums";
import JobsModel from "../models/jobs";
import UsersModel from "../models/users";
import ErrorEvent from "../libs/error";
// import { uploadFromUrl } from "../libs/aws";
/**
* Classe permettant la gestion des albums d'un utilisateur
*/
class Albums extends Pages {
/**
* Méthode permettant de remplacer certains cartactères par leur équivalents html
* @param {String} str
*
* @return {String}
*/
static replaceSpecialChars(str) {
if (!str) {
return "";
}
let final = str.toString();
const find = ["&", "<", ">"];
const replace = ["&amp;", "&lt;", "&gt;"];
for (let i = 0; i < find.length; i += 1) {
final = final.replace(new RegExp(find[i], "g"), replace[i]);
}
return final;
}
/**
* Méthode permettant de convertir les rows en csv
* @param {Array} rows
*
* @return {string}
*/
static async convertToCsv(rows) {
let data =
"Artiste;Titre;Genre;Styles;Pays;Année;Date de sortie;Format\n\r";
for (let i = 0; i < rows.length; i += 1) {
const {
artists_sort,
title,
genres,
styles,
country,
year,
released,
formats,
} = rows[i];
let format = "";
for (let j = 0; j < formats.length; j += 1) {
format += `${format !== "" ? ", " : ""}${formats[j].name}`;
}
data += `${artists_sort};${title};${genres.join()};${styles.join()};${country};${year};${released};${format}\n\r`;
}
return data;
}
/**
* Méthode permettant de convertir les rows en fichier xls
* @param {Array} rows
*
* @return {Object}
*/
static async convertToXls(rows) {
const wb = new xl.Workbook();
const ws = wb.addWorksheet("MusicTopus");
const headerStyle = wb.createStyle({
font: {
color: "#FFFFFF",
size: 11,
},
fill: {
type: "pattern",
patternType: "solid",
bgColor: "#595959",
fgColor: "#595959",
},
});
const style = wb.createStyle({
font: {
color: "#000000",
size: 11,
},
numberFormat: "0000",
});
const header = [
"Artiste",
"Titre",
"Genre",
"Styles",
"Pays",
"Année",
"Date de sortie",
"Format",
];
for (let i = 0; i < header.length; i += 1) {
ws.cell(1, i + 1)
.string(header[i])
.style(headerStyle);
}
for (let i = 0; i < rows.length; i += 1) {
const currentRow = i + 2;
const {
artists_sort,
title,
genres,
styles,
country,
year,
released,
formats,
} = rows[i];
let format = "";
for (let j = 0; j < formats.length; j += 1) {
format += `${format !== "" ? ", " : ""}${formats[j].name}`;
}
ws.cell(currentRow, 1).string(artists_sort).style(style);
ws.cell(currentRow, 2).string(title).style(style);
ws.cell(currentRow, 3).string(genres.join()).style(style);
ws.cell(currentRow, 4).string(styles.join()).style(style);
if (country) {
ws.cell(currentRow, 5).string(country).style(style);
}
if (year) {
ws.cell(currentRow, 6).number(year).style(style);
}
if (released) {
ws.cell(currentRow, 7)
.date(momenttz.tz(released, "Europe/Paris").hour(12))
.style({ numberFormat: "dd/mm/yyyy" });
}
ws.cell(currentRow, 8).string(format).style(style);
}
return wb;
}
/**
* Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
* @param {Array} rows
*
* @return {string}
*/
static async convertToXml(rows) {
let data = '<?xml version="1.0" encoding="UTF-8"?>\n\r<albums>';
for (let i = 0; i < rows.length; i += 1) {
const {
discogsId,
year,
released,
uri,
artists,
artists_sort,
labels,
series,
companies,
formats,
title,
country,
notes,
identifiers,
videos,
genres,
styles,
tracklist,
extraartists,
images,
thumb,
} = rows[i];
let artistsList = "";
let labelList = "";
let serieList = "";
let companiesList = "";
let formatsList = "";
let identifiersList = "";
let videosList = "";
let genresList = "";
let stylesList = "";
let tracklistList = "";
let extraartistsList = "";
let imagesList = "";
for (let j = 0; j < artists.length; j += 1) {
artistsList += `<artist>
<name>${Albums.replaceSpecialChars(artists[j].name)}</name>
<anv>${Albums.replaceSpecialChars(artists[j].anv)}</anv>
<join>${Albums.replaceSpecialChars(artists[j].join)}</join>
<role>${Albums.replaceSpecialChars(artists[j].role)}</role>
<tracks>${Albums.replaceSpecialChars(
artists[j].tracks
)}</tracks>
<id>${Albums.replaceSpecialChars(artists[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
artists[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
artists[j].thumbnail_url
)}</thumbnail_url>
</artist>`;
}
for (let j = 0; j < labels.length; j += 1) {
labelList += `<label>
<name>${Albums.replaceSpecialChars(labels[j].name)}</name>
<catno>${Albums.replaceSpecialChars(labels[j].catno)}</catno>
<entity_type>${Albums.replaceSpecialChars(
labels[j].entity_type
)}</entity_type>
<entity_type_name>${Albums.replaceSpecialChars(
labels[j].entity_type
)}</entity_type_name>
<id>${Albums.replaceSpecialChars(labels[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
labels[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
labels[j].thumbnail_url
)}</thumbnail_url>
</label>
`;
}
for (let j = 0; j < series.length; j += 1) {
serieList += `<serie>
<name>${Albums.replaceSpecialChars(series[j].name)}</name>
<catno>${Albums.replaceSpecialChars(series[j].catno)}</catno>
<entity_type>${Albums.replaceSpecialChars(
series[j].entity_type
)}</entity_type>
<entity_type_name>${Albums.replaceSpecialChars(
series[j].entity_type_name
)}</entity_type_name>
<id>${Albums.replaceSpecialChars(series[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
series[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
series[j].thumbnail_url
)}</thumbnail_url>
</serie>
`;
}
for (let j = 0; j < companies.length; j += 1) {
companiesList += `<company>
<name>${Albums.replaceSpecialChars(companies[j].name)}</name>
<catno>${Albums.replaceSpecialChars(companies[j].catno)}</catno>
<entity_type>${Albums.replaceSpecialChars(
companies[j].entity_type
)}</entity_type>
<entity_type_name>${Albums.replaceSpecialChars(
companies[j].entity_type_name
)}</entity_type_name>
<id>${Albums.replaceSpecialChars(companies[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
companies[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
companies[j].thumbnail_url
)}</thumbnail_url>
</company>
`;
}
for (let j = 0; j < formats.length; j += 1) {
let descriptions = "";
if (formats[j].descriptions) {
for (
let k = 0;
k < formats[j].descriptions.length;
k += 1
) {
descriptions += `<description>${formats[j].descriptions[k]}</description>
`;
}
}
formatsList += `<format>
<name>${Albums.replaceSpecialChars(formats[j].name)}</name>
<qte>${Albums.replaceSpecialChars(formats[j].qty)}</qte>
<text>${Albums.replaceSpecialChars(formats[j].text)}</text>
<descriptions>
${descriptions}
</descriptions>
</format>
`;
}
for (let j = 0; j < identifiers.length; j += 1) {
identifiersList += `<identifier>
<type>${Albums.replaceSpecialChars(identifiers[j].type)}</type>
<value>${Albums.replaceSpecialChars(
identifiers[j].value
)}</value>
<description>${Albums.replaceSpecialChars(
identifiers[j].description
)}</description>
</identifier>
`;
}
for (let j = 0; j < videos.length; j += 1) {
videosList += `<video embed="${videos[j].embed}">
<uri>${Albums.replaceSpecialChars(videos[j].uri)}</uri>
<title>${Albums.replaceSpecialChars(videos[j].title)}</title>
<description>${Albums.replaceSpecialChars(
videos[j].description
)}</description>
<duration>${Albums.replaceSpecialChars(
videos[j].duration
)}</duration>
</video>
`;
}
for (let j = 0; j < genres.length; j += 1) {
genresList += `<genre>${Albums.replaceSpecialChars(
genres[j]
)}</genre>
`;
}
for (let j = 0; j < styles.length; j += 1) {
stylesList += `<style>${Albums.replaceSpecialChars(
styles[j]
)}</style>
`;
}
for (let j = 0; j < tracklist.length; j += 1) {
tracklistList += `<tracklist position="${
tracklist[j].position
}" type="${tracklist[j].type_}" duration="${
tracklist[j].duration
}">
${Albums.replaceSpecialChars(tracklist[j].title)}
</tracklist>
`;
}
for (let j = 0; j < extraartists.length; j += 1) {
extraartistsList += `<extraartist>
<name>${Albums.replaceSpecialChars(extraartists[j].name)}</name>
<anv>${Albums.replaceSpecialChars(extraartists[j].anv)}</anv>
<join>${Albums.replaceSpecialChars(extraartists[j].join)}</join>
<role>${Albums.replaceSpecialChars(extraartists[j].role)}</role>
<tracks>${Albums.replaceSpecialChars(
extraartists[j].tracks
)}</tracks>
<id>${Albums.replaceSpecialChars(extraartists[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
extraartists[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
extraartists[j].thumbnail_url
)}</thumbnail_url>
</extraartist>
`;
}
for (let j = 0; j < images.length; j += 1) {
imagesList += `<image type="${images[j].type}" width="${
images[j].width
}" height="${images[j].height}">
<uri>${Albums.replaceSpecialChars(images[j].uri)}</uri>
<resource_url>${Albums.replaceSpecialChars(
images[j].resource_url
)}</resource_url>
<uri150>${Albums.replaceSpecialChars(
images[j].resource_url
)}</uri150>
</image>
`;
}
data += `
<album>
<discogId>${discogsId}</discogId>
<title>${Albums.replaceSpecialChars(title)}</title>
<artists_sort>${Albums.replaceSpecialChars(artists_sort)}</artists_sort>
<artists>
${artistsList}
</artists>
<year>${year}</year>
<country>${Albums.replaceSpecialChars(country)}</country>
<released>${released}</released>
<uri>${uri}</uri>
<thumb>${thumb}</thumb>
<labels>
${labelList}
</labels>
<series>
${serieList}
</series>
<companies>
${companiesList}
</companies>
<formats>
${formatsList}
</formats>
<notes>${Albums.replaceSpecialChars(notes)}</notes>
<identifiers>
${identifiersList}
</identifiers>
<videos>
${videosList}
</videos>
<genres>
${genresList}
</genres>
<styles>
${stylesList}
</styles>
<tracklist>
${tracklistList}
</tracklist>
<extraartists>
${extraartistsList}
</extraartists>
<images>
${imagesList}
</images>
</album>`;
}
return `${data}</albums>`;
}
/**
* Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
* @param {Array} rows
*
* @return {string}
*/
static async convertToMusicTopus(rows) {
let data = "itemId;createdAt;updatedAt\n\r";
for (let i = 0; i < rows.length; i += 1) {
const { discogsId, createdAt, updatedAt } = rows[i];
data += `${discogsId};${createdAt};${updatedAt}\n\r`;
}
data += "v1.0";
return data;
}
/**
* Méthode permettant d'ajouter un album dans une collection
* @param {Object} req
@ -482,7 +32,18 @@ class Albums extends Pages {
const album = new AlbumsModel(data);
return album.save();
await album.save();
const jobData = {
model: "Albums",
id: album._id,
};
const job = new JobsModel(jobData);
job.save();
return album;
}
/**
@ -593,13 +154,13 @@ class Albums extends Pages {
switch (exportFormat) {
case "csv":
return Albums.convertToCsv(rows);
return Export.convertToCsv(rows);
case "xls":
return Albums.convertToXls(rows);
return Export.convertToXls(rows);
case "xml":
return Albums.convertToXml(rows);
return Export.convertToXml(rows);
case "musictopus":
return Albums.convertToMusicTopus(rows);
return Export.convertToMusicTopus(rows);
case "json":
default:
return {

453
src/middleware/Export.js Normal file
View file

@ -0,0 +1,453 @@
import momenttz from "moment-timezone";
import xl from "excel4node";
class Export {
/**
* Méthode permettant de remplacer certains cartactères par leur équivalents html
* @param {String} str
*
* @return {String}
*/
static replaceSpecialChars(str) {
if (!str) {
return "";
}
let final = str.toString();
const find = ["&", "<", ">"];
const replace = ["&amp;", "&lt;", "&gt;"];
for (let i = 0; i < find.length; i += 1) {
final = final.replace(new RegExp(find[i], "g"), replace[i]);
}
return final;
}
/**
* Méthode permettant de convertir les rows en csv
* @param {Array} rows
*
* @return {string}
*/
static async convertToCsv(rows) {
let data =
"Artiste;Titre;Genre;Styles;Pays;Année;Date de sortie;Format\n\r";
for (let i = 0; i < rows.length; i += 1) {
const {
artists_sort,
title,
genres,
styles,
country,
year,
released,
formats,
} = rows[i];
let format = "";
for (let j = 0; j < formats.length; j += 1) {
format += `${format !== "" ? ", " : ""}${formats[j].name}`;
}
data += `${artists_sort};${title};${genres.join()};${styles.join()};${country};${year};${released};${format}\n\r`;
}
return data;
}
/**
* Méthode permettant de convertir les rows en fichier xls
* @param {Array} rows
*
* @return {Object}
*/
static async convertToXls(rows) {
const wb = new xl.Workbook();
const ws = wb.addWorksheet("MusicTopus");
const headerStyle = wb.createStyle({
font: {
color: "#FFFFFF",
size: 11,
},
fill: {
type: "pattern",
patternType: "solid",
bgColor: "#595959",
fgColor: "#595959",
},
});
const style = wb.createStyle({
font: {
color: "#000000",
size: 11,
},
numberFormat: "0000",
});
const header = [
"Artiste",
"Titre",
"Genre",
"Styles",
"Pays",
"Année",
"Date de sortie",
"Format",
];
for (let i = 0; i < header.length; i += 1) {
ws.cell(1, i + 1)
.string(header[i])
.style(headerStyle);
}
for (let i = 0; i < rows.length; i += 1) {
const currentRow = i + 2;
const {
artists_sort,
title,
genres,
styles,
country,
year,
released,
formats,
} = rows[i];
let format = "";
for (let j = 0; j < formats.length; j += 1) {
format += `${format !== "" ? ", " : ""}${formats[j].name}`;
}
ws.cell(currentRow, 1).string(artists_sort).style(style);
ws.cell(currentRow, 2).string(title).style(style);
ws.cell(currentRow, 3).string(genres.join()).style(style);
ws.cell(currentRow, 4).string(styles.join()).style(style);
if (country) {
ws.cell(currentRow, 5).string(country).style(style);
}
if (year) {
ws.cell(currentRow, 6).number(year).style(style);
}
if (released) {
ws.cell(currentRow, 7)
.date(momenttz.tz(released, "Europe/Paris").hour(12))
.style({ numberFormat: "dd/mm/yyyy" });
}
ws.cell(currentRow, 8).string(format).style(style);
}
return wb;
}
/**
* Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
* @param {Array} rows
*
* @return {string}
*/
static async convertToXml(rows) {
let data = '<?xml version="1.0" encoding="UTF-8"?>\n\r<albums>';
for (let i = 0; i < rows.length; i += 1) {
const {
discogsId,
year,
released,
uri,
artists,
artists_sort,
labels,
series,
companies,
formats,
title,
country,
notes,
identifiers,
videos,
genres,
styles,
tracklist,
extraartists,
images,
thumb,
} = rows[i];
let artistsList = "";
let labelList = "";
let serieList = "";
let companiesList = "";
let formatsList = "";
let identifiersList = "";
let videosList = "";
let genresList = "";
let stylesList = "";
let tracklistList = "";
let extraartistsList = "";
let imagesList = "";
for (let j = 0; j < artists.length; j += 1) {
artistsList += `<artist>
<name>${Export.replaceSpecialChars(artists[j].name)}</name>
<anv>${Export.replaceSpecialChars(artists[j].anv)}</anv>
<join>${Export.replaceSpecialChars(artists[j].join)}</join>
<role>${Export.replaceSpecialChars(artists[j].role)}</role>
<tracks>${Export.replaceSpecialChars(artists[j].tracks)}</tracks>
<id>${Export.replaceSpecialChars(artists[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
artists[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
artists[j].thumbnail_url
)}</thumbnail_url>
</artist>`;
}
for (let j = 0; j < labels.length; j += 1) {
labelList += `<label>
<name>${Export.replaceSpecialChars(labels[j].name)}</name>
<catno>${Export.replaceSpecialChars(labels[j].catno)}</catno>
<entity_type>${Export.replaceSpecialChars(
labels[j].entity_type
)}</entity_type>
<entity_type_name>${Export.replaceSpecialChars(
labels[j].entity_type
)}</entity_type_name>
<id>${Export.replaceSpecialChars(labels[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
labels[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
labels[j].thumbnail_url
)}</thumbnail_url>
</label>
`;
}
for (let j = 0; j < series.length; j += 1) {
serieList += `<serie>
<name>${Export.replaceSpecialChars(series[j].name)}</name>
<catno>${Export.replaceSpecialChars(series[j].catno)}</catno>
<entity_type>${Export.replaceSpecialChars(
series[j].entity_type
)}</entity_type>
<entity_type_name>${Export.replaceSpecialChars(
series[j].entity_type_name
)}</entity_type_name>
<id>${Export.replaceSpecialChars(series[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
series[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
series[j].thumbnail_url
)}</thumbnail_url>
</serie>
`;
}
for (let j = 0; j < companies.length; j += 1) {
companiesList += `<company>
<name>${Export.replaceSpecialChars(companies[j].name)}</name>
<catno>${Export.replaceSpecialChars(companies[j].catno)}</catno>
<entity_type>${Export.replaceSpecialChars(
companies[j].entity_type
)}</entity_type>
<entity_type_name>${Export.replaceSpecialChars(
companies[j].entity_type_name
)}</entity_type_name>
<id>${Export.replaceSpecialChars(companies[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
companies[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
companies[j].thumbnail_url
)}</thumbnail_url>
</company>
`;
}
for (let j = 0; j < formats.length; j += 1) {
let descriptions = "";
if (formats[j].descriptions) {
for (
let k = 0;
k < formats[j].descriptions.length;
k += 1
) {
descriptions += `<description>${formats[j].descriptions[k]}</description>
`;
}
}
formatsList += `<format>
<name>${Export.replaceSpecialChars(formats[j].name)}</name>
<qte>${Export.replaceSpecialChars(formats[j].qty)}</qte>
<text>${Export.replaceSpecialChars(formats[j].text)}</text>
<descriptions>
${descriptions}
</descriptions>
</format>
`;
}
for (let j = 0; j < identifiers.length; j += 1) {
identifiersList += `<identifier>
<type>${Export.replaceSpecialChars(identifiers[j].type)}</type>
<value>${Export.replaceSpecialChars(identifiers[j].value)}</value>
<description>${Export.replaceSpecialChars(
identifiers[j].description
)}</description>
</identifier>
`;
}
for (let j = 0; j < videos.length; j += 1) {
videosList += `<video embed="${videos[j].embed}">
<uri>${Export.replaceSpecialChars(videos[j].uri)}</uri>
<title>${Export.replaceSpecialChars(videos[j].title)}</title>
<description>${Export.replaceSpecialChars(
videos[j].description
)}</description>
<duration>${Export.replaceSpecialChars(
videos[j].duration
)}</duration>
</video>
`;
}
for (let j = 0; j < genres.length; j += 1) {
genresList += `<genre>${Export.replaceSpecialChars(
genres[j]
)}</genre>
`;
}
for (let j = 0; j < styles.length; j += 1) {
stylesList += `<style>${Export.replaceSpecialChars(
styles[j]
)}</style>
`;
}
for (let j = 0; j < tracklist.length; j += 1) {
tracklistList += `<tracklist position="${
tracklist[j].position
}" type="${tracklist[j].type_}" duration="${
tracklist[j].duration
}">
${Export.replaceSpecialChars(tracklist[j].title)}
</tracklist>
`;
}
for (let j = 0; j < extraartists.length; j += 1) {
extraartistsList += `<extraartist>
<name>${Export.replaceSpecialChars(extraartists[j].name)}</name>
<anv>${Export.replaceSpecialChars(extraartists[j].anv)}</anv>
<join>${Export.replaceSpecialChars(extraartists[j].join)}</join>
<role>${Export.replaceSpecialChars(extraartists[j].role)}</role>
<tracks>${Export.replaceSpecialChars(
extraartists[j].tracks
)}</tracks>
<id>${Export.replaceSpecialChars(extraartists[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
extraartists[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
extraartists[j].thumbnail_url
)}</thumbnail_url>
</extraartist>
`;
}
for (let j = 0; j < images.length; j += 1) {
imagesList += `<image type="${images[j].type}" width="${
images[j].width
}" height="${images[j].height}">
<uri>${Export.replaceSpecialChars(images[j].uri)}</uri>
<resource_url>${Export.replaceSpecialChars(
images[j].resource_url
)}</resource_url>
<uri150>${Export.replaceSpecialChars(
images[j].resource_url
)}</uri150>
</image>
`;
}
data += `
<album>
<discogId>${discogsId}</discogId>
<title>${Export.replaceSpecialChars(title)}</title>
<artists_sort>${Export.replaceSpecialChars(artists_sort)}</artists_sort>
<artists>
${artistsList}
</artists>
<year>${year}</year>
<country>${Export.replaceSpecialChars(country)}</country>
<released>${released}</released>
<uri>${uri}</uri>
<thumb>${thumb}</thumb>
<labels>
${labelList}
</labels>
<series>
${serieList}
</series>
<companies>
${companiesList}
</companies>
<formats>
${formatsList}
</formats>
<notes>${Export.replaceSpecialChars(notes)}</notes>
<identifiers>
${identifiersList}
</identifiers>
<videos>
${videosList}
</videos>
<genres>
${genresList}
</genres>
<styles>
${stylesList}
</styles>
<tracklist>
${tracklistList}
</tracklist>
<extraartists>
${extraartistsList}
</extraartists>
<images>
${imagesList}
</images>
</album>`;
}
return `${data}</albums>`;
}
/**
* Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
* @param {Array} rows
*
* @return {string}
*/
static async convertToMusicTopus(rows) {
let data = "itemId;createdAt;updatedAt\n\r";
for (let i = 0; i < rows.length; i += 1) {
const { discogsId, createdAt, updatedAt } = rows[i];
data += `${discogsId};${createdAt};${updatedAt}\n\r`;
}
data += "v1.0";
return data;
}
}
export default Export;

128
src/middleware/Jobs.js Normal file
View file

@ -0,0 +1,128 @@
/* eslint-disable no-await-in-loop */
import ErrorEvent from "../libs/error";
import { uploadFromUrl } from "../libs/aws";
import { getAlbumDetails } from "../helpers";
import JobsModel from "../models/jobs";
import AlbumsModel from "../models/albums";
class Jobs {
/**
* Méthode permettant de télécharger toute les images d'un album
* @param {ObjectId} itemId
*/
static async importAlbumAssets(itemId) {
const album = await AlbumsModel.findById(itemId);
if (!album) {
throw new ErrorEvent(
404,
"Item non trouvé",
`L'album avant l'id ${itemId} n'existe plus dans la collection`
);
}
const item = await getAlbumDetails(album.discogsId);
if (!item) {
throw new ErrorEvent(
404,
"Erreur de communication",
"Erreur lors de la récupération des informations sur Discogs"
);
}
if (item.thumb) {
album.thumb = await uploadFromUrl(item.thumb);
album.thumbType = "local";
}
const { images } = item;
if (images && images.length > 0) {
for (let i = 0; i < images.length; i += 1) {
images[i].uri150 = await uploadFromUrl(images[i].uri150);
images[i].uri = await uploadFromUrl(images[i].uri);
}
}
album.images = images;
await album.save();
return true;
}
/**
* Point d'entrée
* @param {String} state
*
* @return {Object}
*/
async run(state = "NEW") {
const job = await JobsModel.findOne({
state,
tries: {
$lte: 5,
},
});
if (!job) {
return { message: "All jobs done" };
}
job.state = "IN-PROGRESS";
await job.save();
try {
switch (job.model) {
case "Albums":
await Jobs.importAlbumAssets(job.id);
break;
default:
throw new ErrorEvent(
500,
"Job inconnu",
`Le job avec l'id ${job._id} n'est pas un job valide`
);
}
job.state = "SUCCESS";
await job.save();
return this.run(state);
} catch (err) {
job.state = "ERROR";
job.lastTry = new Date();
job.lastErrorMessage = err.message;
job.tries += 1;
await job.save();
throw err;
}
}
/**
* Méthode permettant de créer tous les jobs
*
* @return {Object}
*/
static async populate() {
const albums = await AlbumsModel.find();
for (let i = 0; i < albums.length; i += 1) {
const jobData = {
model: "Albums",
id: albums[i]._id,
};
const job = new JobsModel(jobData);
await job.save();
}
return { message: `${albums.length} jobs ajouté à la file d'attente` };
}
}
export default Jobs;

1
src/models/albums.js
View file

@ -29,6 +29,7 @@ const AlbumSchema = new mongoose.Schema(
extraartists: Array,
images: Array,
thumb: String,
thumbType: String,
},
{ timestamps: true }
);

24
src/models/jobs.js Normal file
View file

@ -0,0 +1,24 @@
import mongoose from "mongoose";
const { Schema } = mongoose;
const JobSchema = new mongoose.Schema(
{
model: String,
id: Schema.Types.ObjectId,
state: {
type: String,
enum: ["NEW", "IN-PROGRESS", "ERROR", "SUCCESS"],
default: "NEW",
},
lastTry: Date,
lastErrorMessage: String,
tries: {
type: Number,
default: 0,
},
},
{ timestamps: true }
);
export default mongoose.model("Jobs", JobSchema);

40
src/routes/jobs.js Normal file
View file

@ -0,0 +1,40 @@
import express from "express";
import passport from "passport";
import Jobs from "../middleware/Jobs";
// eslint-disable-next-line new-cap
const router = express.Router();
router.route("/").get(
passport.authenticate(["jobs"], {
session: false,
}),
async (req, res, next) => {
try {
const job = new Jobs();
const data = await job.run(req.query.state);
return res.status(200).json(data).end();
} catch (err) {
return next(err);
}
}
);
router.route("/populate").get(
passport.authenticate(["jobs"], {
session: false,
}),
async (req, res, next) => {
try {
const data = await Jobs.populate();
return res.status(200).json(data).end();
} catch (err) {
return next(err);
}
}
);
export default router;