README.md

@@ -62,7 +62,7 @@ Le site est accessible sur [http://localhost:PORT](http://localhost:PORT).
 
 #### Standalone
 
-Pour la version standalone je vous conseille de faire un script embarquant les variables d'environnement que vous souhaitez modifier :
+Pour la version standalone je vous conseille de faire un script embarquant les variables d'environnement que vous souhaitez modifier ([voir à la fin pour la liste des variables](#env-file)) :
 
 ```bash
 #! /bin/bash
@@ -184,6 +184,26 @@ server {
 
 Une fois le vhost activé (lien symbolique dans le dossier site-enable) et nginx rechargé votre site sera alors accessible en https.
 
+### Jobs
+
+Par défaut toute les images des albums sont affichées depuis Discogs. Cependant avec les temps les urls deviennent invalides. Pour éviter cela lors de l'ajout d'un album à votre collection un job est créé. Ce job a pour rôle de stocker les images sur un bucket s3.
+
+Pour lancer les jobs il faut mettre en place une tâche cron qui sera éxécutée toute les heures (par exemple).
+
+Exemple de crontab :
+```crontab
+0 * * * * curl 'http://localhost:3001/jobs' \
+    -H 'JOBS_HEADER_KEY: JOBS_HEADER_VALUE' \
+    -H 'Accept: application/json'
+30 * * * * curl 'http://localhost:3001/jobs?state=ERROR' \
+    -H 'JOBS_HEADER_KEY: JOBS_HEADER_VALUE' \
+    -H 'Accept: application/json'
+```
+
+N'oubliez pas de remplacer `localhost:30001`, `JOBS_HEADER_KEY` et `JOBS_HEADER_VALUE` par les bonnes valeurs.
+
+La première ligne permet de parcourir tous les nouveaux jobs alors que la seconde permet de relancer les jobs en erreurs (après 5 tentatives le job est marqué comme définitivement perdu).
+
 ### Fichier .env {#env-file}
 
 Voici la liste des variables configurables :
@@ -198,10 +218,17 @@ FORMSPREE_ID # Id du formulaire formspree pour la page "nous-contacter"
 MATOMO_URL # Url vers l'instance matomo (exemple: https://analytics.darkou.fr/)
 MATOMO_ID # Id du site sur votre instance matomo (exemple: 1)
 SITE_NAME # Nom du site (utilisé dans le titre des pages)
+AWS_ACCESS_KEY_ID # Clé d'accès AWS
+AWS_SECRET_ACCESS_KEY # Clé secrète AWS
+S3_ENDPOINT # Url de l'instance aws (s3.fr-par.scw.cloud pour scaleway france par exemple)
+S3_SIGNATURE # Version de la signature AWS (s3v4 pour scaleway par exemple)
+S3_BASEFOLDER # Nom du sous dossier dans lequel seront mis les pochettes des albums
+S3_BUCKET # Nom du bucket
+JOBS_HEADER_KEY # Nom du header utilisé pour l'identification des tâches cron (par exemple musictopus)
+JOBS_HEADER_VALUE # Valeur de la clé
 ```
 
 ## Contributeurs
 
 - Damien Broqua (développeur principal du projet)
 - Brunus (Logo et fournisseur d'idées :wink: )
-

docker-compose.yml.dev

@@ -28,6 +28,14 @@ services:
       MATOMO_URL: ${MATOMO_URL}
       MATOMO_ID: ${MATOMO_ID}
       SITE_NAME: ${SITE_NAME}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      S3_BASEFOLDER: ${S3_BASEFOLDER}
+      S3_BUCKET: ${S3_BUCKET}
+      S3_ENDPOINT: ${S3_ENDPOINT}
+      S3_SIGNATURE: ${S3_SIGNATURE}
+      JOBS_HEADER_KEY: ${JOBS_HEADER_KEY}
+      JOBS_HEADER_VALUE: ${JOBS_HEADER_VALUE}
     networks:
       - musictopus
   musictopus-db:

docker-compose.yml.prod

@@ -28,6 +28,14 @@ services:
       MATOMO_URL: ${MATOMO_URL}
       MATOMO_ID: ${MATOMO_ID}
       SITE_NAME: ${SITE_NAME}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      S3_BASEFOLDER: ${S3_BASEFOLDER}
+      S3_BUCKET: ${S3_BUCKET}
+      S3_ENDPOINT: ${S3_ENDPOINT}
+      S3_SIGNATURE: ${S3_SIGNATURE}
+      JOBS_HEADER_KEY: ${JOBS_HEADER_KEY}
+      JOBS_HEADER_VALUE: ${JOBS_HEADER_VALUE}
     networks:
       - musictopus
   musictopus-db:

package.json

@@ -41,6 +41,7 @@
     "@babel/cli": "^7.17.0",
     "@babel/core": "^7.17.2",
     "@babel/preset-env": "^7.16.11",
+    "aws-sdk": "^2.1110.0",
     "axios": "^0.26.0",
     "connect-ensure-login": "^0.1.1",
     "connect-flash": "^0.1.1",
@@ -60,10 +61,12 @@
     "mongoose-unique-validator": "^3.0.0",
     "npm-run-all": "^4.1.5",
     "passport": "^0.5.2",
+    "passport-custom": "^1.1.1",
     "passport-http": "^0.3.0",
     "passport-local": "^1.0.0",
     "rimraf": "^3.0.2",
     "sass": "^1.49.7",
+    "uuid": "^8.3.2",
     "vue": "^3.2.31"
   },
   "nodemonConfig": {

src/app.js

@@ -7,6 +7,8 @@ import flash from "connect-flash";
 import session from "express-session";
 import MongoStore from "connect-mongo";
 
+import passportConfig from "./libs/passport";
+
 import config, { env, mongoDbUri, secret } from "./config";
 
 import { isXhr } from "./helpers";
@@ -15,15 +17,13 @@ import indexRouter from "./routes";
 import maCollectionRouter from "./routes/ma-collection";
 import collectionRouter from "./routes/collection";
 
+import importJobsRouter from "./routes/jobs";
+
 import importAlbumRouterApiV1 from "./routes/api/v1/albums";
 import importSearchRouterApiV1 from "./routes/api/v1/search";
 import importMeRouterApiV1 from "./routes/api/v1/me";
 
-// Mongoose schema init
+passportConfig(passport);
-require("./models/users");
-require("./models/albums");
-
-require("./libs/passport")(passport);
 
 mongoose
     .connect(mongoDbUri, { useNewUrlParser: true, useUnifiedTopology: true })
@@ -46,10 +46,10 @@ const sess = {
 
 const app = express();
 
-app.use(express.json());
-app.use(express.urlencoded({ extended: false }));
 app.use(cookieParser());
 app.use(flash());
+app.use(express.json({ limit: "50mb" }));
+app.use(express.urlencoded({ extended: false, limit: "50mb" }));
 
 app.use(session(sess));
 
@@ -85,6 +85,7 @@ app.use(
 app.use("/", indexRouter);
 app.use("/ma-collection", maCollectionRouter);
 app.use("/collection", collectionRouter);
+app.use("/jobs", importJobsRouter);
 app.use("/api/v1/albums", importAlbumRouterApiV1);
 app.use("/api/v1/search", importSearchRouterApiV1);
 app.use("/api/v1/me", importMeRouterApiV1);

src/config/index.js

@@ -8,4 +8,13 @@ module.exports = {
     matomoUrl: process.env.MATOMO_URL || "",
     matomoId: process.env.MATOMO_ID || "",
     siteName: process.env.SITE_NAME || "MusicTopus",
+    awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID,
+    awsSecretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+    s3BaseFolder: process.env.S3_BASEFOLDER || "dev",
+    s3Bucket: process.env.S3_BUCKET || "musictopus",
+    s3Endpoint: process.env.S3_ENDPOINT || "s3.fr-par.scw.cloud",
+    s3Signature: process.env.S3_SIGNATURE || "s3v4",
+    jobsHeaderKey: process.env.JOBS_HEADER_KEY || "musictopus",
+    jobsHeaderValue:
+        process.env.JOBS_HEADER_VALUE || "ooYee9xok7eigo2shiePohyoGh1eepew",
 };

src/libs/aws.js

@@ -0,0 +1,72 @@
+import AWS from "aws-sdk";
+import fs from "fs";
+import path from "path";
+import axios from "axios";
+import { v4 as uuid } from "uuid";
+
+import {
+    awsAccessKeyId,
+    awsSecretAccessKey,
+    s3BaseFolder,
+    s3Endpoint,
+    s3Bucket,
+    s3Signature,
+} from "../config";
+
+AWS.config.update({
+    accessKeyId: awsAccessKeyId,
+    secretAccessKey: awsSecretAccessKey,
+});
+/**
+ * Fonction permettant de stocker un fichier local sur S3
+ * @param {String} filename
+ * @param {String} file
+ * @param {Boolean} deleteFile
+ *
+ * @return {String}
+ */
+export const uploadFromFile = async (filename, file, deleteFile = false) => {
+    const data = await fs.readFileSync(file);
+
+    const base64data = Buffer.from(data, "binary");
+    const dest = path.join(s3BaseFolder, filename);
+
+    const s3 = new AWS.S3({
+        endpoint: s3Endpoint,
+        signatureVersion: s3Signature,
+    });
+
+    await s3
+        .putObject({
+            Bucket: s3Bucket,
+            Key: dest,
+            Body: base64data,
+            ACL: "public-read",
+        })
+        .promise();
+
+    if (deleteFile) {
+        fs.unlinkSync(file);
+    }
+
+    return `https://${s3Bucket}.${s3Endpoint}/${dest}`;
+};
+
+/**
+ * Fonction permettant de stocker un fichier provenant d'une URL sur S3
+ * @param {String} url
+ *
+ * @return {String}
+ */
+export const uploadFromUrl = async (url) => {
+    const filename = `${uuid()}.jpg`;
+    const file = `/tmp/${filename}`;
+
+    const { data } = await axios.get(url, { responseType: "arraybuffer" });
+
+    fs.writeFileSync(file, data);
+
+    return uploadFromFile(filename, file, true);
+
+    // return s3Object;
+};

src/libs/passport.js

@@ -1,11 +1,13 @@
 /* eslint-disable func-names */
-const mongoose = require("mongoose");
+import { Strategy as LocalStrategy } from "passport-local";
-const LocalStrategy = require("passport-local").Strategy;
+import { BasicStrategy } from "passport-http";
-const { BasicStrategy } = require("passport-http");
+import { Strategy as CustomStrategy } from "passport-custom";
 
-const Users = mongoose.model("Users");
+import Users from "../models/users";
 
-module.exports = function (passport) {
+import { jobsHeaderKey, jobsHeaderValue } from "../config";
+
+export default (passport) => {
     passport.serializeUser((user, done) => {
         done(null, user);
     });
@@ -55,4 +57,17 @@ module.exports = function (passport) {
                 .catch(done);
         })
     );
+    passport.use(
+        "jobs",
+        new CustomStrategy((req, next) => {
+            const apiKey = req.headers[jobsHeaderKey];
+
+            if (apiKey === jobsHeaderValue) {
+                return next(null, {
+                    username: "jobs",
+                });
+            }
+            return next(null, false, "Oops! Identifiants incorrects");
+        })
+    );
 };

src/middleware/Albums.js

@@ -1,468 +1,18 @@
 import moment from "moment";
-import momenttz from "moment-timezone";
-import xl from "excel4node";
 
 import Pages from "./Pages";
+import Export from "./Export";
 
 import AlbumsModel from "../models/albums";
+import JobsModel from "../models/jobs";
 import UsersModel from "../models/users";
 import ErrorEvent from "../libs/error";
+// import { uploadFromUrl } from "../libs/aws";
 
 /**
  * Classe permettant la gestion des albums d'un utilisateur
  */
 class Albums extends Pages {
-    /**
-     * Méthode permettant de remplacer certains cartactères par leur équivalents html
-     * @param {String} str
-     *
-     * @return {String}
-     */
-    static replaceSpecialChars(str) {
-        if (!str) {
-            return "";
-        }
-        let final = str.toString();
-        const find = ["&", "<", ">"];
-        const replace = ["&amp;", "&lt;", "&gt;"];
-
-        for (let i = 0; i < find.length; i += 1) {
-            final = final.replace(new RegExp(find[i], "g"), replace[i]);
-        }
-
-        return final;
-    }
-
-    /**
-     * Méthode permettant de convertir les rows en csv
-     * @param {Array} rows
-     *
-     * @return {string}
-     */
-    static async convertToCsv(rows) {
-        let data =
-            "Artiste;Titre;Genre;Styles;Pays;Année;Date de sortie;Format\n\r";
-
-        for (let i = 0; i < rows.length; i += 1) {
-            const {
-                artists_sort,
-                title,
-                genres,
-                styles,
-                country,
-                year,
-                released,
-                formats,
-            } = rows[i];
-
-            let format = "";
-            for (let j = 0; j < formats.length; j += 1) {
-                format += `${format !== "" ? ", " : ""}${formats[j].name}`;
-            }
-
-            data += `${artists_sort};${title};${genres.join()};${styles.join()};${country};${year};${released};${format}\n\r`;
-        }
-
-        return data;
-    }
-
-    /**
-     * Méthode permettant de convertir les rows en fichier xls
-     * @param {Array} rows
-     *
-     * @return {Object}
-     */
-    static async convertToXls(rows) {
-        const wb = new xl.Workbook();
-        const ws = wb.addWorksheet("MusicTopus");
-
-        const headerStyle = wb.createStyle({
-            font: {
-                color: "#FFFFFF",
-                size: 11,
-            },
-            fill: {
-                type: "pattern",
-                patternType: "solid",
-                bgColor: "#595959",
-                fgColor: "#595959",
-            },
-        });
-        const style = wb.createStyle({
-            font: {
-                color: "#000000",
-                size: 11,
-            },
-            numberFormat: "0000",
-        });
-
-        const header = [
-            "Artiste",
-            "Titre",
-            "Genre",
-            "Styles",
-            "Pays",
-            "Année",
-            "Date de sortie",
-            "Format",
-        ];
-        for (let i = 0; i < header.length; i += 1) {
-            ws.cell(1, i + 1)
-                .string(header[i])
-                .style(headerStyle);
-        }
-
-        for (let i = 0; i < rows.length; i += 1) {
-            const currentRow = i + 2;
-            const {
-                artists_sort,
-                title,
-                genres,
-                styles,
-                country,
-                year,
-                released,
-                formats,
-            } = rows[i];
-
-            let format = "";
-            for (let j = 0; j < formats.length; j += 1) {
-                format += `${format !== "" ? ", " : ""}${formats[j].name}`;
-            }
-
-            ws.cell(currentRow, 1).string(artists_sort).style(style);
-            ws.cell(currentRow, 2).string(title).style(style);
-            ws.cell(currentRow, 3).string(genres.join()).style(style);
-            ws.cell(currentRow, 4).string(styles.join()).style(style);
-            if (country) {
-                ws.cell(currentRow, 5).string(country).style(style);
-            }
-            if (year) {
-                ws.cell(currentRow, 6).number(year).style(style);
-            }
-            if (released) {
-                ws.cell(currentRow, 7)
-                    .date(momenttz.tz(released, "Europe/Paris").hour(12))
-                    .style({ numberFormat: "dd/mm/yyyy" });
-            }
-            ws.cell(currentRow, 8).string(format).style(style);
-        }
-
-        return wb;
-    }
-
-    /**
-     * Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
-     * @param {Array} rows
-     *
-     * @return {string}
-     */
-    static async convertToXml(rows) {
-        let data = '<?xml version="1.0" encoding="UTF-8"?>\n\r<albums>';
-
-        for (let i = 0; i < rows.length; i += 1) {
-            const {
-                discogsId,
-                year,
-                released,
-                uri,
-                artists,
-                artists_sort,
-                labels,
-                series,
-                companies,
-                formats,
-                title,
-                country,
-                notes,
-                identifiers,
-                videos,
-                genres,
-                styles,
-                tracklist,
-                extraartists,
-                images,
-                thumb,
-            } = rows[i];
-
-            let artistsList = "";
-            let labelList = "";
-            let serieList = "";
-            let companiesList = "";
-            let formatsList = "";
-            let identifiersList = "";
-            let videosList = "";
-            let genresList = "";
-            let stylesList = "";
-            let tracklistList = "";
-            let extraartistsList = "";
-            let imagesList = "";
-
-            for (let j = 0; j < artists.length; j += 1) {
-                artistsList += `<artist>
-                <name>${Albums.replaceSpecialChars(artists[j].name)}</name>
-                <anv>${Albums.replaceSpecialChars(artists[j].anv)}</anv>
-                <join>${Albums.replaceSpecialChars(artists[j].join)}</join>
-                <role>${Albums.replaceSpecialChars(artists[j].role)}</role>
-                <tracks>${Albums.replaceSpecialChars(
-                    artists[j].tracks
-                )}</tracks>
-                <id>${Albums.replaceSpecialChars(artists[j].id)}</id>
-                <resource_url>${Albums.replaceSpecialChars(
-                    artists[j].resource_url
-                )}</resource_url>
-                <thumbnail_url>${Albums.replaceSpecialChars(
-                    artists[j].thumbnail_url
-                )}</thumbnail_url>
-            </artist>`;
-            }
-
-            for (let j = 0; j < labels.length; j += 1) {
-                labelList += `<label>
-                <name>${Albums.replaceSpecialChars(labels[j].name)}</name>
-                <catno>${Albums.replaceSpecialChars(labels[j].catno)}</catno>
-                <entity_type>${Albums.replaceSpecialChars(
-                    labels[j].entity_type
-                )}</entity_type>
-                <entity_type_name>${Albums.replaceSpecialChars(
-                    labels[j].entity_type
-                )}</entity_type_name>
-                <id>${Albums.replaceSpecialChars(labels[j].id)}</id>
-                <resource_url>${Albums.replaceSpecialChars(
-                    labels[j].resource_url
-                )}</resource_url>
-                <thumbnail_url>${Albums.replaceSpecialChars(
-                    labels[j].thumbnail_url
-                )}</thumbnail_url>
-            </label>
-            `;
-            }
-
-            for (let j = 0; j < series.length; j += 1) {
-                serieList += `<serie>
-                <name>${Albums.replaceSpecialChars(series[j].name)}</name>
-                <catno>${Albums.replaceSpecialChars(series[j].catno)}</catno>
-                <entity_type>${Albums.replaceSpecialChars(
-                    series[j].entity_type
-                )}</entity_type>
-                <entity_type_name>${Albums.replaceSpecialChars(
-                    series[j].entity_type_name
-                )}</entity_type_name>
-                <id>${Albums.replaceSpecialChars(series[j].id)}</id>
-                <resource_url>${Albums.replaceSpecialChars(
-                    series[j].resource_url
-                )}</resource_url>
-                <thumbnail_url>${Albums.replaceSpecialChars(
-                    series[j].thumbnail_url
-                )}</thumbnail_url>
-            </serie>
-            `;
-            }
-
-            for (let j = 0; j < companies.length; j += 1) {
-                companiesList += `<company>
-                <name>${Albums.replaceSpecialChars(companies[j].name)}</name>
-                <catno>${Albums.replaceSpecialChars(companies[j].catno)}</catno>
-                <entity_type>${Albums.replaceSpecialChars(
-                    companies[j].entity_type
-                )}</entity_type>
-                <entity_type_name>${Albums.replaceSpecialChars(
-                    companies[j].entity_type_name
-                )}</entity_type_name>
-                <id>${Albums.replaceSpecialChars(companies[j].id)}</id>
-                <resource_url>${Albums.replaceSpecialChars(
-                    companies[j].resource_url
-                )}</resource_url>
-                <thumbnail_url>${Albums.replaceSpecialChars(
-                    companies[j].thumbnail_url
-                )}</thumbnail_url>
-            </company>
-            `;
-            }
-
-            for (let j = 0; j < formats.length; j += 1) {
-                let descriptions = "";
-                if (formats[j].descriptions) {
-                    for (
-                        let k = 0;
-                        k < formats[j].descriptions.length;
-                        k += 1
-                    ) {
-                        descriptions += `<description>${formats[j].descriptions[k]}</description>
-                    `;
-                    }
-                }
-                formatsList += `<format>
-                <name>${Albums.replaceSpecialChars(formats[j].name)}</name>
-                <qte>${Albums.replaceSpecialChars(formats[j].qty)}</qte>
-                <text>${Albums.replaceSpecialChars(formats[j].text)}</text>
-                <descriptions>
-                    ${descriptions}
-                </descriptions>
-            </format>
-            `;
-            }
-
-            for (let j = 0; j < identifiers.length; j += 1) {
-                identifiersList += `<identifier>
-                <type>${Albums.replaceSpecialChars(identifiers[j].type)}</type>
-                <value>${Albums.replaceSpecialChars(
-                    identifiers[j].value
-                )}</value>
-                <description>${Albums.replaceSpecialChars(
-                    identifiers[j].description
-                )}</description>
-            </identifier>
-            `;
-            }
-
-            for (let j = 0; j < videos.length; j += 1) {
-                videosList += `<video embed="${videos[j].embed}">
-                <uri>${Albums.replaceSpecialChars(videos[j].uri)}</uri>
-                <title>${Albums.replaceSpecialChars(videos[j].title)}</title>
-                <description>${Albums.replaceSpecialChars(
-                    videos[j].description
-                )}</description>
-                <duration>${Albums.replaceSpecialChars(
-                    videos[j].duration
-                )}</duration>
-            </video>
-            `;
-            }
-
-            for (let j = 0; j < genres.length; j += 1) {
-                genresList += `<genre>${Albums.replaceSpecialChars(
-                    genres[j]
-                )}</genre>
-            `;
-            }
-
-            for (let j = 0; j < styles.length; j += 1) {
-                stylesList += `<style>${Albums.replaceSpecialChars(
-                    styles[j]
-                )}</style>
-            `;
-            }
-
-            for (let j = 0; j < tracklist.length; j += 1) {
-                tracklistList += `<tracklist position="${
-                    tracklist[j].position
-                }" type="${tracklist[j].type_}" duration="${
-                    tracklist[j].duration
-                }">
-                ${Albums.replaceSpecialChars(tracklist[j].title)}
-            </tracklist>
-            `;
-            }
-
-            for (let j = 0; j < extraartists.length; j += 1) {
-                extraartistsList += `<extraartist>
-                <name>${Albums.replaceSpecialChars(extraartists[j].name)}</name>
-                <anv>${Albums.replaceSpecialChars(extraartists[j].anv)}</anv>
-                <join>${Albums.replaceSpecialChars(extraartists[j].join)}</join>
-                <role>${Albums.replaceSpecialChars(extraartists[j].role)}</role>
-                <tracks>${Albums.replaceSpecialChars(
-                    extraartists[j].tracks
-                )}</tracks>
-                <id>${Albums.replaceSpecialChars(extraartists[j].id)}</id>
-                <resource_url>${Albums.replaceSpecialChars(
-                    extraartists[j].resource_url
-                )}</resource_url>
-                <thumbnail_url>${Albums.replaceSpecialChars(
-                    extraartists[j].thumbnail_url
-                )}</thumbnail_url>
-            </extraartist>
-            `;
-            }
-
-            for (let j = 0; j < images.length; j += 1) {
-                imagesList += `<image type="${images[j].type}" width="${
-                    images[j].width
-                }" height="${images[j].height}">
-                <uri>${Albums.replaceSpecialChars(images[j].uri)}</uri>
-                <resource_url>${Albums.replaceSpecialChars(
-                    images[j].resource_url
-                )}</resource_url>
-                <uri150>${Albums.replaceSpecialChars(
-                    images[j].resource_url
-                )}</uri150>
-            </image>
-            `;
-            }
-
-            data += `
-    <album>
-        <discogId>${discogsId}</discogId>
-        <title>${Albums.replaceSpecialChars(title)}</title>
-        <artists_sort>${Albums.replaceSpecialChars(artists_sort)}</artists_sort>
-        <artists>
-            ${artistsList}
-        </artists>
-        <year>${year}</year>
-        <country>${Albums.replaceSpecialChars(country)}</country>
-        <released>${released}</released>
-        <uri>${uri}</uri>
-        <thumb>${thumb}</thumb>
-        <labels>
-            ${labelList}
-        </labels>
-        <series>
-            ${serieList}
-        </series>
-        <companies>
-            ${companiesList}
-        </companies>
-        <formats>
-            ${formatsList}
-        </formats>
-        <notes>${Albums.replaceSpecialChars(notes)}</notes>
-        <identifiers>
-            ${identifiersList}
-        </identifiers>
-        <videos>
-            ${videosList}
-        </videos>
-        <genres>
-            ${genresList}
-        </genres>
-        <styles>
-            ${stylesList}
-        </styles>
-        <tracklist>
-            ${tracklistList}
-        </tracklist>
-        <extraartists>
-            ${extraartistsList}
-        </extraartists>
-        <images>
-            ${imagesList}
-        </images>
-    </album>`;
-        }
-
-        return `${data}</albums>`;
-    }
-
-    /**
-     * Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
-     * @param {Array} rows
-     *
-     * @return {string}
-     */
-    static async convertToMusicTopus(rows) {
-        let data = "itemId;createdAt;updatedAt\n\r";
-
-        for (let i = 0; i < rows.length; i += 1) {
-            const { discogsId, createdAt, updatedAt } = rows[i];
-
-            data += `${discogsId};${createdAt};${updatedAt}\n\r`;
-        }
-
-        data += "v1.0";
-
-        return data;
-    }
-
     /**
      * Méthode permettant d'ajouter un album dans une collection
      * @param {Object} req
@@ -482,7 +32,18 @@ class Albums extends Pages {
 
         const album = new AlbumsModel(data);
 
-        return album.save();
+        await album.save();
+
+        const jobData = {
+            model: "Albums",
+            id: album._id,
+        };
+
+        const job = new JobsModel(jobData);
+
+        job.save();
+
+        return album;
     }
 
     /**
@@ -593,13 +154,13 @@ class Albums extends Pages {
 
         switch (exportFormat) {
             case "csv":
-                return Albums.convertToCsv(rows);
+                return Export.convertToCsv(rows);
             case "xls":
-                return Albums.convertToXls(rows);
+                return Export.convertToXls(rows);
             case "xml":
-                return Albums.convertToXml(rows);
+                return Export.convertToXml(rows);
             case "musictopus":
-                return Albums.convertToMusicTopus(rows);
+                return Export.convertToMusicTopus(rows);
             case "json":
             default:
                 return {

src/middleware/Export.js

@@ -0,0 +1,453 @@
+import momenttz from "moment-timezone";
+import xl from "excel4node";
+
+class Export {
+    /**
+     * Méthode permettant de remplacer certains cartactères par leur équivalents html
+     * @param {String} str
+     *
+     * @return {String}
+     */
+    static replaceSpecialChars(str) {
+        if (!str) {
+            return "";
+        }
+        let final = str.toString();
+        const find = ["&", "<", ">"];
+        const replace = ["&amp;", "&lt;", "&gt;"];
+
+        for (let i = 0; i < find.length; i += 1) {
+            final = final.replace(new RegExp(find[i], "g"), replace[i]);
+        }
+
+        return final;
+    }
+
+    /**
+     * Méthode permettant de convertir les rows en csv
+     * @param {Array} rows
+     *
+     * @return {string}
+     */
+    static async convertToCsv(rows) {
+        let data =
+            "Artiste;Titre;Genre;Styles;Pays;Année;Date de sortie;Format\n\r";
+
+        for (let i = 0; i < rows.length; i += 1) {
+            const {
+                artists_sort,
+                title,
+                genres,
+                styles,
+                country,
+                year,
+                released,
+                formats,
+            } = rows[i];
+
+            let format = "";
+            for (let j = 0; j < formats.length; j += 1) {
+                format += `${format !== "" ? ", " : ""}${formats[j].name}`;
+            }
+
+            data += `${artists_sort};${title};${genres.join()};${styles.join()};${country};${year};${released};${format}\n\r`;
+        }
+
+        return data;
+    }
+
+    /**
+     * Méthode permettant de convertir les rows en fichier xls
+     * @param {Array} rows
+     *
+     * @return {Object}
+     */
+    static async convertToXls(rows) {
+        const wb = new xl.Workbook();
+        const ws = wb.addWorksheet("MusicTopus");
+
+        const headerStyle = wb.createStyle({
+            font: {
+                color: "#FFFFFF",
+                size: 11,
+            },
+            fill: {
+                type: "pattern",
+                patternType: "solid",
+                bgColor: "#595959",
+                fgColor: "#595959",
+            },
+        });
+        const style = wb.createStyle({
+            font: {
+                color: "#000000",
+                size: 11,
+            },
+            numberFormat: "0000",
+        });
+
+        const header = [
+            "Artiste",
+            "Titre",
+            "Genre",
+            "Styles",
+            "Pays",
+            "Année",
+            "Date de sortie",
+            "Format",
+        ];
+        for (let i = 0; i < header.length; i += 1) {
+            ws.cell(1, i + 1)
+                .string(header[i])
+                .style(headerStyle);
+        }
+
+        for (let i = 0; i < rows.length; i += 1) {
+            const currentRow = i + 2;
+            const {
+                artists_sort,
+                title,
+                genres,
+                styles,
+                country,
+                year,
+                released,
+                formats,
+            } = rows[i];
+
+            let format = "";
+            for (let j = 0; j < formats.length; j += 1) {
+                format += `${format !== "" ? ", " : ""}${formats[j].name}`;
+            }
+
+            ws.cell(currentRow, 1).string(artists_sort).style(style);
+            ws.cell(currentRow, 2).string(title).style(style);
+            ws.cell(currentRow, 3).string(genres.join()).style(style);
+            ws.cell(currentRow, 4).string(styles.join()).style(style);
+            if (country) {
+                ws.cell(currentRow, 5).string(country).style(style);
+            }
+            if (year) {
+                ws.cell(currentRow, 6).number(year).style(style);
+            }
+            if (released) {
+                ws.cell(currentRow, 7)
+                    .date(momenttz.tz(released, "Europe/Paris").hour(12))
+                    .style({ numberFormat: "dd/mm/yyyy" });
+            }
+            ws.cell(currentRow, 8).string(format).style(style);
+        }
+
+        return wb;
+    }
+
+    /**
+     * Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
+     * @param {Array} rows
+     *
+     * @return {string}
+     */
+    static async convertToXml(rows) {
+        let data = '<?xml version="1.0" encoding="UTF-8"?>\n\r<albums>';
+
+        for (let i = 0; i < rows.length; i += 1) {
+            const {
+                discogsId,
+                year,
+                released,
+                uri,
+                artists,
+                artists_sort,
+                labels,
+                series,
+                companies,
+                formats,
+                title,
+                country,
+                notes,
+                identifiers,
+                videos,
+                genres,
+                styles,
+                tracklist,
+                extraartists,
+                images,
+                thumb,
+            } = rows[i];
+
+            let artistsList = "";
+            let labelList = "";
+            let serieList = "";
+            let companiesList = "";
+            let formatsList = "";
+            let identifiersList = "";
+            let videosList = "";
+            let genresList = "";
+            let stylesList = "";
+            let tracklistList = "";
+            let extraartistsList = "";
+            let imagesList = "";
+
+            for (let j = 0; j < artists.length; j += 1) {
+                artistsList += `<artist>
+            <name>${Export.replaceSpecialChars(artists[j].name)}</name>
+            <anv>${Export.replaceSpecialChars(artists[j].anv)}</anv>
+            <join>${Export.replaceSpecialChars(artists[j].join)}</join>
+            <role>${Export.replaceSpecialChars(artists[j].role)}</role>
+            <tracks>${Export.replaceSpecialChars(artists[j].tracks)}</tracks>
+            <id>${Export.replaceSpecialChars(artists[j].id)}</id>
+            <resource_url>${Export.replaceSpecialChars(
+                artists[j].resource_url
+            )}</resource_url>
+            <thumbnail_url>${Export.replaceSpecialChars(
+                artists[j].thumbnail_url
+            )}</thumbnail_url>
+        </artist>`;
+            }
+
+            for (let j = 0; j < labels.length; j += 1) {
+                labelList += `<label>
+            <name>${Export.replaceSpecialChars(labels[j].name)}</name>
+            <catno>${Export.replaceSpecialChars(labels[j].catno)}</catno>
+            <entity_type>${Export.replaceSpecialChars(
+                labels[j].entity_type
+            )}</entity_type>
+            <entity_type_name>${Export.replaceSpecialChars(
+                labels[j].entity_type
+            )}</entity_type_name>
+            <id>${Export.replaceSpecialChars(labels[j].id)}</id>
+            <resource_url>${Export.replaceSpecialChars(
+                labels[j].resource_url
+            )}</resource_url>
+            <thumbnail_url>${Export.replaceSpecialChars(
+                labels[j].thumbnail_url
+            )}</thumbnail_url>
+        </label>
+        `;
+            }
+
+            for (let j = 0; j < series.length; j += 1) {
+                serieList += `<serie>
+            <name>${Export.replaceSpecialChars(series[j].name)}</name>
+            <catno>${Export.replaceSpecialChars(series[j].catno)}</catno>
+            <entity_type>${Export.replaceSpecialChars(
+                series[j].entity_type
+            )}</entity_type>
+            <entity_type_name>${Export.replaceSpecialChars(
+                series[j].entity_type_name
+            )}</entity_type_name>
+            <id>${Export.replaceSpecialChars(series[j].id)}</id>
+            <resource_url>${Export.replaceSpecialChars(
+                series[j].resource_url
+            )}</resource_url>
+            <thumbnail_url>${Export.replaceSpecialChars(
+                series[j].thumbnail_url
+            )}</thumbnail_url>
+        </serie>
+        `;
+            }
+
+            for (let j = 0; j < companies.length; j += 1) {
+                companiesList += `<company>
+            <name>${Export.replaceSpecialChars(companies[j].name)}</name>
+            <catno>${Export.replaceSpecialChars(companies[j].catno)}</catno>
+            <entity_type>${Export.replaceSpecialChars(
+                companies[j].entity_type
+            )}</entity_type>
+            <entity_type_name>${Export.replaceSpecialChars(
+                companies[j].entity_type_name
+            )}</entity_type_name>
+            <id>${Export.replaceSpecialChars(companies[j].id)}</id>
+            <resource_url>${Export.replaceSpecialChars(
+                companies[j].resource_url
+            )}</resource_url>
+            <thumbnail_url>${Export.replaceSpecialChars(
+                companies[j].thumbnail_url
+            )}</thumbnail_url>
+        </company>
+        `;
+            }
+
+            for (let j = 0; j < formats.length; j += 1) {
+                let descriptions = "";
+                if (formats[j].descriptions) {
+                    for (
+                        let k = 0;
+                        k < formats[j].descriptions.length;
+                        k += 1
+                    ) {
+                        descriptions += `<description>${formats[j].descriptions[k]}</description>
+                `;
+                    }
+                }
+                formatsList += `<format>
+            <name>${Export.replaceSpecialChars(formats[j].name)}</name>
+            <qte>${Export.replaceSpecialChars(formats[j].qty)}</qte>
+            <text>${Export.replaceSpecialChars(formats[j].text)}</text>
+            <descriptions>
+                ${descriptions}
+            </descriptions>
+        </format>
+        `;
+            }
+
+            for (let j = 0; j < identifiers.length; j += 1) {
+                identifiersList += `<identifier>
+            <type>${Export.replaceSpecialChars(identifiers[j].type)}</type>
+            <value>${Export.replaceSpecialChars(identifiers[j].value)}</value>
+            <description>${Export.replaceSpecialChars(
+                identifiers[j].description
+            )}</description>
+        </identifier>
+        `;
+            }
+
+            for (let j = 0; j < videos.length; j += 1) {
+                videosList += `<video embed="${videos[j].embed}">
+            <uri>${Export.replaceSpecialChars(videos[j].uri)}</uri>
+            <title>${Export.replaceSpecialChars(videos[j].title)}</title>
+            <description>${Export.replaceSpecialChars(
+                videos[j].description
+            )}</description>
+            <duration>${Export.replaceSpecialChars(
+                videos[j].duration
+            )}</duration>
+        </video>
+        `;
+            }
+
+            for (let j = 0; j < genres.length; j += 1) {
+                genresList += `<genre>${Export.replaceSpecialChars(
+                    genres[j]
+                )}</genre>
+        `;
+            }
+
+            for (let j = 0; j < styles.length; j += 1) {
+                stylesList += `<style>${Export.replaceSpecialChars(
+                    styles[j]
+                )}</style>
+        `;
+            }
+
+            for (let j = 0; j < tracklist.length; j += 1) {
+                tracklistList += `<tracklist position="${
+                    tracklist[j].position
+                }" type="${tracklist[j].type_}" duration="${
+                    tracklist[j].duration
+                }">
+            ${Export.replaceSpecialChars(tracklist[j].title)}
+        </tracklist>
+        `;
+            }
+
+            for (let j = 0; j < extraartists.length; j += 1) {
+                extraartistsList += `<extraartist>
+            <name>${Export.replaceSpecialChars(extraartists[j].name)}</name>
+            <anv>${Export.replaceSpecialChars(extraartists[j].anv)}</anv>
+            <join>${Export.replaceSpecialChars(extraartists[j].join)}</join>
+            <role>${Export.replaceSpecialChars(extraartists[j].role)}</role>
+            <tracks>${Export.replaceSpecialChars(
+                extraartists[j].tracks
+            )}</tracks>
+            <id>${Export.replaceSpecialChars(extraartists[j].id)}</id>
+            <resource_url>${Export.replaceSpecialChars(
+                extraartists[j].resource_url
+            )}</resource_url>
+            <thumbnail_url>${Export.replaceSpecialChars(
+                extraartists[j].thumbnail_url
+            )}</thumbnail_url>
+        </extraartist>
+        `;
+            }
+
+            for (let j = 0; j < images.length; j += 1) {
+                imagesList += `<image type="${images[j].type}" width="${
+                    images[j].width
+                }" height="${images[j].height}">
+            <uri>${Export.replaceSpecialChars(images[j].uri)}</uri>
+            <resource_url>${Export.replaceSpecialChars(
+                images[j].resource_url
+            )}</resource_url>
+            <uri150>${Export.replaceSpecialChars(
+                images[j].resource_url
+            )}</uri150>
+        </image>
+        `;
+            }
+
+            data += `
+<album>
+    <discogId>${discogsId}</discogId>
+    <title>${Export.replaceSpecialChars(title)}</title>
+    <artists_sort>${Export.replaceSpecialChars(artists_sort)}</artists_sort>
+    <artists>
+        ${artistsList}
+    </artists>
+    <year>${year}</year>
+    <country>${Export.replaceSpecialChars(country)}</country>
+    <released>${released}</released>
+    <uri>${uri}</uri>
+    <thumb>${thumb}</thumb>
+    <labels>
+        ${labelList}
+    </labels>
+    <series>
+        ${serieList}
+    </series>
+    <companies>
+        ${companiesList}
+    </companies>
+    <formats>
+        ${formatsList}
+    </formats>
+    <notes>${Export.replaceSpecialChars(notes)}</notes>
+    <identifiers>
+        ${identifiersList}
+    </identifiers>
+    <videos>
+        ${videosList}
+    </videos>
+    <genres>
+        ${genresList}
+    </genres>
+    <styles>
+        ${stylesList}
+    </styles>
+    <tracklist>
+        ${tracklistList}
+    </tracklist>
+    <extraartists>
+        ${extraartistsList}
+    </extraartists>
+    <images>
+        ${imagesList}
+    </images>
+</album>`;
+        }
+
+        return `${data}</albums>`;
+    }
+
+    /**
+     * Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
+     * @param {Array} rows
+     *
+     * @return {string}
+     */
+    static async convertToMusicTopus(rows) {
+        let data = "itemId;createdAt;updatedAt\n\r";
+
+        for (let i = 0; i < rows.length; i += 1) {
+            const { discogsId, createdAt, updatedAt } = rows[i];
+
+            data += `${discogsId};${createdAt};${updatedAt}\n\r`;
+        }
+
+        data += "v1.0";
+
+        return data;
+    }
+}
+
+export default Export;

src/middleware/Jobs.js

@@ -0,0 +1,128 @@
+/* eslint-disable no-await-in-loop */
+import ErrorEvent from "../libs/error";
+import { uploadFromUrl } from "../libs/aws";
+import { getAlbumDetails } from "../helpers";
+
+import JobsModel from "../models/jobs";
+import AlbumsModel from "../models/albums";
+
+class Jobs {
+    /**
+     * Méthode permettant de télécharger toute les images d'un album
+     * @param {ObjectId} itemId
+     */
+    static async importAlbumAssets(itemId) {
+        const album = await AlbumsModel.findById(itemId);
+
+        if (!album) {
+            throw new ErrorEvent(
+                404,
+                "Item non trouvé",
+                `L'album avant l'id ${itemId} n'existe plus dans la collection`
+            );
+        }
+
+        const item = await getAlbumDetails(album.discogsId);
+
+        if (!item) {
+            throw new ErrorEvent(
+                404,
+                "Erreur de communication",
+                "Erreur lors de la récupération des informations sur Discogs"
+            );
+        }
+
+        if (item.thumb) {
+            album.thumb = await uploadFromUrl(item.thumb);
+            album.thumbType = "local";
+        }
+        const { images } = item;
+        if (images && images.length > 0) {
+            for (let i = 0; i < images.length; i += 1) {
+                images[i].uri150 = await uploadFromUrl(images[i].uri150);
+                images[i].uri = await uploadFromUrl(images[i].uri);
+            }
+        }
+
+        album.images = images;
+
+        await album.save();
+
+        return true;
+    }
+
+    /**
+     * Point d'entrée
+     * @param {String} state
+     *
+     * @return {Object}
+     */
+    async run(state = "NEW") {
+        const job = await JobsModel.findOne({
+            state,
+            tries: {
+                $lte: 5,
+            },
+        });
+
+        if (!job) {
+            return { message: "All jobs done" };
+        }
+
+        job.state = "IN-PROGRESS";
+
+        await job.save();
+
+        try {
+            switch (job.model) {
+                case "Albums":
+                    await Jobs.importAlbumAssets(job.id);
+                    break;
+                default:
+                    throw new ErrorEvent(
+                        500,
+                        "Job inconnu",
+                        `Le job avec l'id ${job._id} n'est pas un job valide`
+                    );
+            }
+
+            job.state = "SUCCESS";
+
+            await job.save();
+
+            return this.run(state);
+        } catch (err) {
+            job.state = "ERROR";
+            job.lastTry = new Date();
+            job.lastErrorMessage = err.message;
+            job.tries += 1;
+
+            await job.save();
+
+            throw err;
+        }
+    }
+
+    /**
+     * Méthode permettant de créer tous les jobs
+     *
+     * @return {Object}
+     */
+    static async populate() {
+        const albums = await AlbumsModel.find();
+
+        for (let i = 0; i < albums.length; i += 1) {
+            const jobData = {
+                model: "Albums",
+                id: albums[i]._id,
+            };
+
+            const job = new JobsModel(jobData);
+            await job.save();
+        }
+
+        return { message: `${albums.length} jobs ajouté à la file d'attente` };
+    }
+}
+
+export default Jobs;

src/models/albums.js

@@ -29,6 +29,7 @@ const AlbumSchema = new mongoose.Schema(
         extraartists: Array,
         images: Array,
         thumb: String,
+        thumbType: String,
     },
     { timestamps: true }
 );

src/models/jobs.js

@@ -0,0 +1,24 @@
+import mongoose from "mongoose";
+
+const { Schema } = mongoose;
+
+const JobSchema = new mongoose.Schema(
+    {
+        model: String,
+        id: Schema.Types.ObjectId,
+        state: {
+            type: String,
+            enum: ["NEW", "IN-PROGRESS", "ERROR", "SUCCESS"],
+            default: "NEW",
+        },
+        lastTry: Date,
+        lastErrorMessage: String,
+        tries: {
+            type: Number,
+            default: 0,
+        },
+    },
+    { timestamps: true }
+);
+
+export default mongoose.model("Jobs", JobSchema);

src/routes/jobs.js

@@ -0,0 +1,40 @@
+import express from "express";
+import passport from "passport";
+
+import Jobs from "../middleware/Jobs";
+
+// eslint-disable-next-line new-cap
+const router = express.Router();
+
+router.route("/").get(
+    passport.authenticate(["jobs"], {
+        session: false,
+    }),
+    async (req, res, next) => {
+        try {
+            const job = new Jobs();
+            const data = await job.run(req.query.state);
+
+            return res.status(200).json(data).end();
+        } catch (err) {
+            return next(err);
+        }
+    }
+);
+
+router.route("/populate").get(
+    passport.authenticate(["jobs"], {
+        session: false,
+    }),
+    async (req, res, next) => {
+        try {
+            const data = await Jobs.populate();
+
+            return res.status(200).json(data).end();
+        } catch (err) {
+            return next(err);
+        }
+    }
+);
+
+export default router;