diff --git a/README.md b/README.md index 1da497b..e05aa15 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Le site est accessible sur [http://localhost:PORT](http://localhost:PORT). #### Standalone -Pour la version standalone je vous conseille de faire un script embarquant les variables d'environnement que vous souhaitez modifier : +Pour la version standalone je vous conseille de faire un script embarquant les variables d'environnement que vous souhaitez modifier ([voir à la fin pour la liste des variables](#env-file)) : ```bash #! /bin/bash @@ -184,6 +184,26 @@ server { Une fois le vhost activé (lien symbolique dans le dossier site-enable) et nginx rechargé votre site sera alors accessible en https. +### Jobs + +Par défaut toute les images des albums sont affichées depuis Discogs. Cependant avec les temps les urls deviennent invalides. Pour éviter cela lors de l'ajout d'un album à votre collection un job est créé. Ce job a pour rôle de stocker les images sur un bucket s3. + +Pour lancer les jobs il faut mettre en place une tâche cron qui sera éxécutée toute les heures (par exemple). + +Exemple de crontab : +```crontab +0 * * * * curl 'http://localhost:3001/jobs' \ + -H 'JOBS_HEADER_KEY: JOBS_HEADER_VALUE' \ + -H 'Accept: application/json' +30 * * * * curl 'http://localhost:3001/jobs?state=ERROR' \ + -H 'JOBS_HEADER_KEY: JOBS_HEADER_VALUE' \ + -H 'Accept: application/json' +``` + +N'oubliez pas de remplacer `localhost:30001`, `JOBS_HEADER_KEY` et `JOBS_HEADER_VALUE` par les bonnes valeurs. + +La première ligne permet de parcourir tous les nouveaux jobs alors que la seconde permet de relancer les jobs en erreurs (après 5 tentatives le job est marqué comme définitivement perdu). + ### Fichier .env {#env-file} Voici la liste des variables configurables : @@ -198,10 +218,17 @@ FORMSPREE_ID # Id du formulaire formspree pour la page "nous-contacter" MATOMO_URL # Url vers l'instance matomo (exemple: https://analytics.darkou.fr/) MATOMO_ID # Id du site sur votre instance matomo (exemple: 1) SITE_NAME # Nom du site (utilisé dans le titre des pages) +AWS_ACCESS_KEY_ID # Clé d'accès AWS +AWS_SECRET_ACCESS_KEY # Clé secrète AWS +S3_ENDPOINT # Url de l'instance aws (s3.fr-par.scw.cloud pour scaleway france par exemple) +S3_SIGNATURE # Version de la signature AWS (s3v4 pour scaleway par exemple) +S3_BASEFOLDER # Nom du sous dossier dans lequel seront mis les pochettes des albums +S3_BUCKET # Nom du bucket +JOBS_HEADER_KEY # Nom du header utilisé pour l'identification des tâches cron (par exemple musictopus) +JOBS_HEADER_VALUE # Valeur de la clé ``` ## Contributeurs - Damien Broqua (développeur principal du projet) - Brunus (Logo et fournisseur d'idées :wink: ) - diff --git a/docker-compose.yml.dev b/docker-compose.yml.dev index 9391624..dbe38b9 100644 --- a/docker-compose.yml.dev +++ b/docker-compose.yml.dev @@ -28,6 +28,14 @@ services: MATOMO_URL: ${MATOMO_URL} MATOMO_ID: ${MATOMO_ID} SITE_NAME: ${SITE_NAME} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + S3_BASEFOLDER: ${S3_BASEFOLDER} + S3_BUCKET: ${S3_BUCKET} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_SIGNATURE: ${S3_SIGNATURE} + JOBS_HEADER_KEY: ${JOBS_HEADER_KEY} + JOBS_HEADER_VALUE: ${JOBS_HEADER_VALUE} networks: - musictopus musictopus-db: diff --git a/docker-compose.yml.prod b/docker-compose.yml.prod index 9077877..c343531 100644 --- a/docker-compose.yml.prod +++ b/docker-compose.yml.prod @@ -28,6 +28,14 @@ services: MATOMO_URL: ${MATOMO_URL} MATOMO_ID: ${MATOMO_ID} SITE_NAME: ${SITE_NAME} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + S3_BASEFOLDER: ${S3_BASEFOLDER} + S3_BUCKET: ${S3_BUCKET} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_SIGNATURE: ${S3_SIGNATURE} + JOBS_HEADER_KEY: ${JOBS_HEADER_KEY} + JOBS_HEADER_VALUE: ${JOBS_HEADER_VALUE} networks: - musictopus musictopus-db: diff --git a/package.json b/package.json index b5aea27..d574778 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "@babel/cli": "^7.17.0", "@babel/core": "^7.17.2", "@babel/preset-env": "^7.16.11", + "aws-sdk": "^2.1110.0", "axios": "^0.26.0", "connect-ensure-login": "^0.1.1", "connect-flash": "^0.1.1", @@ -60,10 +61,12 @@ "mongoose-unique-validator": "^3.0.0", "npm-run-all": "^4.1.5", "passport": "^0.5.2", + "passport-custom": "^1.1.1", "passport-http": "^0.3.0", "passport-local": "^1.0.0", "rimraf": "^3.0.2", "sass": "^1.49.7", + "uuid": "^8.3.2", "vue": "^3.2.31" }, "nodemonConfig": { diff --git a/public/img/404.svg b/public/img/404.svg index b4dbe78..4cd6c0c 100644 --- a/public/img/404.svg +++ b/public/img/404.svg @@ -6,50 +6,50 @@ xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" - viewBox="0 0 168.85766 133.4734" - version="1.1" - id="MusicTopus" + width="100%" height="100%" - width="100%"> + id="MusicTopus" + version="1.1" + viewBox="0 0 168.85766 133.4734"> + offset="0" /> + stop-opacity="0" + offset="1" /> - + x2="103.29" + gradientTransform="translate(-19.041285,-22.505715)" + y1="27.309999" + x1="57.074001" /> + + offset="0" /> + offset="1" /> + transform="translate(-4.0461145,-24.740973)" + id="layer1"> + id="g4845" + transform="matrix(-1,0,0,1,16.909353,13.841748)"> - - - - - + + + + + + style="fill:#ec8479;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#fbb9b8;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> - + + transform="translate(4.0461145,24.740973)" + id="path4" /> + style="opacity:1;fill:#ffaf62;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.41468528;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:fill markers stroke" + rx="6.7384396" + ry="5.6584005" /> + id="path4788" + style="opacity:1;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.41468525;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:fill markers stroke" + rx="1.1466434" + ry="0.96285915" /> + style="fill:#feb6b7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#ffd7d7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#feb6b9;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#dd6258;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#feb6b7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#ffd7d7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#dd6258;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#eea6a7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#feb6b7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#ffd7d7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#dd6258;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#feb6b7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#ffd7d7;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#feb6b9;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#dd6258;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + transform="rotate(19.617168,87.590538,52.720911)" + id="g4740"> + + + + + + + + id="path4744" /> - - - - - - - + cx="105.70718" + id="circle4655" + style="opacity:1;fill:#ffffff;fill-opacity:0.94117647;fill-rule:nonzero;stroke:none;stroke-width:0.14161019;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:fill markers stroke" /> - + + style="fill:#dd6258;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#dd6258;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#dd6258;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#dd6258;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + style="fill:#7f2625;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> - + style="fill:none;stroke:#301818;stroke-width:4.46500015;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" /> + + style="opacity:1;vector-effect:none;fill:#7f2625;fill-opacity:1;stroke:none;stroke-width:1.48535442;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" /> + style="stroke-width:42.36951447"> + id="path2" /> + id="path4-9" /> + transform="matrix(-0.0125514,0,0,0.0125514,96.697579,101.08859)"> + style="fill:#29abe2;stroke-width:79.67237854" /> + style="fill:#ffffff;stroke-width:79.67237854" /> + + - - + id="path4700" /> diff --git a/public/js/main.js b/public/js/main.js index 895683a..f203a63 100644 --- a/public/js/main.js +++ b/public/js/main.js @@ -105,7 +105,7 @@ function switchTheme(e) { document.addEventListener('DOMContentLoaded', () => { const $navbarBurgers = Array.prototype.slice.call(document.querySelectorAll('.navbar-burger'), 0); if ($navbarBurgers.length > 0) { - $navbarBurgers.forEach( el => { + $navbarBurgers.forEach( el => { el.addEventListener('click', () => { const target = el.dataset.target; const $target = document.getElementById(target); diff --git a/sass/index.scss b/sass/index.scss index 3a53fd2..373b302 100644 --- a/sass/index.scss +++ b/sass/index.scss @@ -46,4 +46,4 @@ @import './ajouter-un-album'; @import './collection'; @import './ma-collection-details'; -@import './composants'; \ No newline at end of file +@import './composants'; diff --git a/sass/list.scss b/sass/list.scss index 9dbd03a..fbdc7c2 100644 --- a/sass/list.scss +++ b/sass/list.scss @@ -23,6 +23,12 @@ background-color: var(--default-color); } + &:nth-child(4n), + &:nth-child(4n-1) + { + background-color: var(--default-color); + } + &:first-child, &:nth-child(2) { border-top: 2px solid var(--border-color); diff --git a/src/app.js b/src/app.js index 6d1e507..c8a9574 100644 --- a/src/app.js +++ b/src/app.js @@ -7,6 +7,8 @@ import flash from "connect-flash"; import session from "express-session"; import MongoStore from "connect-mongo"; +import passportConfig from "./libs/passport"; + import config, { env, mongoDbUri, secret } from "./config"; import { isXhr } from "./helpers"; @@ -15,15 +17,13 @@ import indexRouter from "./routes"; import maCollectionRouter from "./routes/ma-collection"; import collectionRouter from "./routes/collection"; +import importJobsRouter from "./routes/jobs"; + import importAlbumRouterApiV1 from "./routes/api/v1/albums"; import importSearchRouterApiV1 from "./routes/api/v1/search"; import importMeRouterApiV1 from "./routes/api/v1/me"; -// Mongoose schema init -require("./models/users"); -require("./models/albums"); - -require("./libs/passport")(passport); +passportConfig(passport); mongoose .connect(mongoDbUri, { useNewUrlParser: true, useUnifiedTopology: true }) @@ -46,10 +46,10 @@ const sess = { const app = express(); -app.use(express.json()); -app.use(express.urlencoded({ extended: false })); app.use(cookieParser()); app.use(flash()); +app.use(express.json({ limit: "50mb" })); +app.use(express.urlencoded({ extended: false, limit: "50mb" })); app.use(session(sess)); @@ -85,6 +85,7 @@ app.use( app.use("/", indexRouter); app.use("/ma-collection", maCollectionRouter); app.use("/collection", collectionRouter); +app.use("/jobs", importJobsRouter); app.use("/api/v1/albums", importAlbumRouterApiV1); app.use("/api/v1/search", importSearchRouterApiV1); app.use("/api/v1/me", importMeRouterApiV1); diff --git a/src/config/index.js b/src/config/index.js index 19f16ea..17b341d 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -8,4 +8,13 @@ module.exports = { matomoUrl: process.env.MATOMO_URL || "", matomoId: process.env.MATOMO_ID || "", siteName: process.env.SITE_NAME || "MusicTopus", + awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID, + awsSecretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + s3BaseFolder: process.env.S3_BASEFOLDER || "dev", + s3Bucket: process.env.S3_BUCKET || "musictopus", + s3Endpoint: process.env.S3_ENDPOINT || "s3.fr-par.scw.cloud", + s3Signature: process.env.S3_SIGNATURE || "s3v4", + jobsHeaderKey: process.env.JOBS_HEADER_KEY || "musictopus", + jobsHeaderValue: + process.env.JOBS_HEADER_VALUE || "ooYee9xok7eigo2shiePohyoGh1eepew", }; diff --git a/src/libs/aws.js b/src/libs/aws.js new file mode 100644 index 0000000..f65521b --- /dev/null +++ b/src/libs/aws.js @@ -0,0 +1,72 @@ +import AWS from "aws-sdk"; +import fs from "fs"; +import path from "path"; +import axios from "axios"; +import { v4 as uuid } from "uuid"; + +import { + awsAccessKeyId, + awsSecretAccessKey, + s3BaseFolder, + s3Endpoint, + s3Bucket, + s3Signature, +} from "../config"; + +AWS.config.update({ + accessKeyId: awsAccessKeyId, + secretAccessKey: awsSecretAccessKey, +}); +/** + * Fonction permettant de stocker un fichier local sur S3 + * @param {String} filename + * @param {String} file + * @param {Boolean} deleteFile + * + * @return {String} + */ +export const uploadFromFile = async (filename, file, deleteFile = false) => { + const data = await fs.readFileSync(file); + + const base64data = Buffer.from(data, "binary"); + const dest = path.join(s3BaseFolder, filename); + + const s3 = new AWS.S3({ + endpoint: s3Endpoint, + signatureVersion: s3Signature, + }); + + await s3 + .putObject({ + Bucket: s3Bucket, + Key: dest, + Body: base64data, + ACL: "public-read", + }) + .promise(); + + if (deleteFile) { + fs.unlinkSync(file); + } + + return `https://${s3Bucket}.${s3Endpoint}/${dest}`; +}; + +/** + * Fonction permettant de stocker un fichier provenant d'une URL sur S3 + * @param {String} url + * + * @return {String} + */ +export const uploadFromUrl = async (url) => { + const filename = `${uuid()}.jpg`; + const file = `/tmp/${filename}`; + + const { data } = await axios.get(url, { responseType: "arraybuffer" }); + + fs.writeFileSync(file, data); + + return uploadFromFile(filename, file, true); + + // return s3Object; +}; diff --git a/src/libs/passport.js b/src/libs/passport.js index af0ab0b..a3ed7fe 100644 --- a/src/libs/passport.js +++ b/src/libs/passport.js @@ -1,11 +1,13 @@ /* eslint-disable func-names */ -const mongoose = require("mongoose"); -const LocalStrategy = require("passport-local").Strategy; -const { BasicStrategy } = require("passport-http"); +import { Strategy as LocalStrategy } from "passport-local"; +import { BasicStrategy } from "passport-http"; +import { Strategy as CustomStrategy } from "passport-custom"; -const Users = mongoose.model("Users"); +import Users from "../models/users"; -module.exports = function (passport) { +import { jobsHeaderKey, jobsHeaderValue } from "../config"; + +export default (passport) => { passport.serializeUser((user, done) => { done(null, user); }); @@ -55,4 +57,17 @@ module.exports = function (passport) { .catch(done); }) ); + passport.use( + "jobs", + new CustomStrategy((req, next) => { + const apiKey = req.headers[jobsHeaderKey]; + + if (apiKey === jobsHeaderValue) { + return next(null, { + username: "jobs", + }); + } + return next(null, false, "Oops! Identifiants incorrects"); + }) + ); }; diff --git a/src/middleware/Albums.js b/src/middleware/Albums.js index 6caa29b..84f5c5d 100644 --- a/src/middleware/Albums.js +++ b/src/middleware/Albums.js @@ -1,468 +1,18 @@ import moment from "moment"; -import momenttz from "moment-timezone"; -import xl from "excel4node"; import Pages from "./Pages"; +import Export from "./Export"; import AlbumsModel from "../models/albums"; +import JobsModel from "../models/jobs"; import UsersModel from "../models/users"; import ErrorEvent from "../libs/error"; +// import { uploadFromUrl } from "../libs/aws"; /** * Classe permettant la gestion des albums d'un utilisateur */ class Albums extends Pages { - /** - * Méthode permettant de remplacer certains cartactères par leur équivalents html - * @param {String} str - * - * @return {String} - */ - static replaceSpecialChars(str) { - if (!str) { - return ""; - } - let final = str.toString(); - const find = ["&", "<", ">"]; - const replace = ["&", "<", ">"]; - - for (let i = 0; i < find.length; i += 1) { - final = final.replace(new RegExp(find[i], "g"), replace[i]); - } - - return final; - } - - /** - * Méthode permettant de convertir les rows en csv - * @param {Array} rows - * - * @return {string} - */ - static async convertToCsv(rows) { - let data = - "Artiste;Titre;Genre;Styles;Pays;Année;Date de sortie;Format\n\r"; - - for (let i = 0; i < rows.length; i += 1) { - const { - artists_sort, - title, - genres, - styles, - country, - year, - released, - formats, - } = rows[i]; - - let format = ""; - for (let j = 0; j < formats.length; j += 1) { - format += `${format !== "" ? ", " : ""}${formats[j].name}`; - } - - data += `${artists_sort};${title};${genres.join()};${styles.join()};${country};${year};${released};${format}\n\r`; - } - - return data; - } - - /** - * Méthode permettant de convertir les rows en fichier xls - * @param {Array} rows - * - * @return {Object} - */ - static async convertToXls(rows) { - const wb = new xl.Workbook(); - const ws = wb.addWorksheet("MusicTopus"); - - const headerStyle = wb.createStyle({ - font: { - color: "#FFFFFF", - size: 11, - }, - fill: { - type: "pattern", - patternType: "solid", - bgColor: "#595959", - fgColor: "#595959", - }, - }); - const style = wb.createStyle({ - font: { - color: "#000000", - size: 11, - }, - numberFormat: "0000", - }); - - const header = [ - "Artiste", - "Titre", - "Genre", - "Styles", - "Pays", - "Année", - "Date de sortie", - "Format", - ]; - for (let i = 0; i < header.length; i += 1) { - ws.cell(1, i + 1) - .string(header[i]) - .style(headerStyle); - } - - for (let i = 0; i < rows.length; i += 1) { - const currentRow = i + 2; - const { - artists_sort, - title, - genres, - styles, - country, - year, - released, - formats, - } = rows[i]; - - let format = ""; - for (let j = 0; j < formats.length; j += 1) { - format += `${format !== "" ? ", " : ""}${formats[j].name}`; - } - - ws.cell(currentRow, 1).string(artists_sort).style(style); - ws.cell(currentRow, 2).string(title).style(style); - ws.cell(currentRow, 3).string(genres.join()).style(style); - ws.cell(currentRow, 4).string(styles.join()).style(style); - if (country) { - ws.cell(currentRow, 5).string(country).style(style); - } - if (year) { - ws.cell(currentRow, 6).number(year).style(style); - } - if (released) { - ws.cell(currentRow, 7) - .date(momenttz.tz(released, "Europe/Paris").hour(12)) - .style({ numberFormat: "dd/mm/yyyy" }); - } - ws.cell(currentRow, 8).string(format).style(style); - } - - return wb; - } - - /** - * Méthode permettant de convertir les rows en csv pour importer dans MusicTopus - * @param {Array} rows - * - * @return {string} - */ - static async convertToXml(rows) { - let data = '\n\r'; - - for (let i = 0; i < rows.length; i += 1) { - const { - discogsId, - year, - released, - uri, - artists, - artists_sort, - labels, - series, - companies, - formats, - title, - country, - notes, - identifiers, - videos, - genres, - styles, - tracklist, - extraartists, - images, - thumb, - } = rows[i]; - - let artistsList = ""; - let labelList = ""; - let serieList = ""; - let companiesList = ""; - let formatsList = ""; - let identifiersList = ""; - let videosList = ""; - let genresList = ""; - let stylesList = ""; - let tracklistList = ""; - let extraartistsList = ""; - let imagesList = ""; - - for (let j = 0; j < artists.length; j += 1) { - artistsList += ` - ${Albums.replaceSpecialChars(artists[j].name)} - ${Albums.replaceSpecialChars(artists[j].anv)} - ${Albums.replaceSpecialChars(artists[j].join)} - ${Albums.replaceSpecialChars(artists[j].role)} - ${Albums.replaceSpecialChars( - artists[j].tracks - )} - ${Albums.replaceSpecialChars(artists[j].id)} - ${Albums.replaceSpecialChars( - artists[j].resource_url - )} - ${Albums.replaceSpecialChars( - artists[j].thumbnail_url - )} - `; - } - - for (let j = 0; j < labels.length; j += 1) { - labelList += ` - `; - } - - for (let j = 0; j < series.length; j += 1) { - serieList += ` - ${Albums.replaceSpecialChars(series[j].name)} - ${Albums.replaceSpecialChars(series[j].catno)} - ${Albums.replaceSpecialChars( - series[j].entity_type - )} - ${Albums.replaceSpecialChars( - series[j].entity_type_name - )} - ${Albums.replaceSpecialChars(series[j].id)} - ${Albums.replaceSpecialChars( - series[j].resource_url - )} - ${Albums.replaceSpecialChars( - series[j].thumbnail_url - )} - - `; - } - - for (let j = 0; j < companies.length; j += 1) { - companiesList += ` - ${Albums.replaceSpecialChars(companies[j].name)} - ${Albums.replaceSpecialChars(companies[j].catno)} - ${Albums.replaceSpecialChars( - companies[j].entity_type - )} - ${Albums.replaceSpecialChars( - companies[j].entity_type_name - )} - ${Albums.replaceSpecialChars(companies[j].id)} - ${Albums.replaceSpecialChars( - companies[j].resource_url - )} - ${Albums.replaceSpecialChars( - companies[j].thumbnail_url - )} - - `; - } - - for (let j = 0; j < formats.length; j += 1) { - let descriptions = ""; - if (formats[j].descriptions) { - for ( - let k = 0; - k < formats[j].descriptions.length; - k += 1 - ) { - descriptions += `${formats[j].descriptions[k]} - `; - } - } - formatsList += ` - ${Albums.replaceSpecialChars(formats[j].name)} - ${Albums.replaceSpecialChars(formats[j].qty)} - ${Albums.replaceSpecialChars(formats[j].text)} - - ${descriptions} - - - `; - } - - for (let j = 0; j < identifiers.length; j += 1) { - identifiersList += ` - ${Albums.replaceSpecialChars(identifiers[j].type)} - ${Albums.replaceSpecialChars( - identifiers[j].value - )} - ${Albums.replaceSpecialChars( - identifiers[j].description - )} - - `; - } - - for (let j = 0; j < videos.length; j += 1) { - videosList += ` - `; - } - - for (let j = 0; j < genres.length; j += 1) { - genresList += `${Albums.replaceSpecialChars( - genres[j] - )} - `; - } - - for (let j = 0; j < styles.length; j += 1) { - stylesList += ` - `; - } - - for (let j = 0; j < tracklist.length; j += 1) { - tracklistList += ` - ${Albums.replaceSpecialChars(tracklist[j].title)} - - `; - } - - for (let j = 0; j < extraartists.length; j += 1) { - extraartistsList += ` - ${Albums.replaceSpecialChars(extraartists[j].name)} - ${Albums.replaceSpecialChars(extraartists[j].anv)} - ${Albums.replaceSpecialChars(extraartists[j].join)} - ${Albums.replaceSpecialChars(extraartists[j].role)} - ${Albums.replaceSpecialChars( - extraartists[j].tracks - )} - ${Albums.replaceSpecialChars(extraartists[j].id)} - ${Albums.replaceSpecialChars( - extraartists[j].resource_url - )} - ${Albums.replaceSpecialChars( - extraartists[j].thumbnail_url - )} - - `; - } - - for (let j = 0; j < images.length; j += 1) { - imagesList += ` - ${Albums.replaceSpecialChars(images[j].uri)} - ${Albums.replaceSpecialChars( - images[j].resource_url - )} - ${Albums.replaceSpecialChars( - images[j].resource_url - )} - - `; - } - - data += ` - - ${discogsId} - ${Albums.replaceSpecialChars(title)} - ${Albums.replaceSpecialChars(artists_sort)} - - ${artistsList} - - ${year} - ${Albums.replaceSpecialChars(country)} - ${released} - ${uri} - ${thumb} - - ${labelList} - - - ${serieList} - - - ${companiesList} - - - ${formatsList} - - ${Albums.replaceSpecialChars(notes)} - - ${identifiersList} - - - ${videosList} - - - ${genresList} - - - ${stylesList} - - - ${tracklistList} - - - ${extraartistsList} - - - ${imagesList} - - `; - } - - return `${data}`; - } - - /** - * Méthode permettant de convertir les rows en csv pour importer dans MusicTopus - * @param {Array} rows - * - * @return {string} - */ - static async convertToMusicTopus(rows) { - let data = "itemId;createdAt;updatedAt\n\r"; - - for (let i = 0; i < rows.length; i += 1) { - const { discogsId, createdAt, updatedAt } = rows[i]; - - data += `${discogsId};${createdAt};${updatedAt}\n\r`; - } - - data += "v1.0"; - - return data; - } - /** * Méthode permettant d'ajouter un album dans une collection * @param {Object} req @@ -482,7 +32,18 @@ class Albums extends Pages { const album = new AlbumsModel(data); - return album.save(); + await album.save(); + + const jobData = { + model: "Albums", + id: album._id, + }; + + const job = new JobsModel(jobData); + + job.save(); + + return album; } /** @@ -593,13 +154,13 @@ class Albums extends Pages { switch (exportFormat) { case "csv": - return Albums.convertToCsv(rows); + return Export.convertToCsv(rows); case "xls": - return Albums.convertToXls(rows); + return Export.convertToXls(rows); case "xml": - return Albums.convertToXml(rows); + return Export.convertToXml(rows); case "musictopus": - return Albums.convertToMusicTopus(rows); + return Export.convertToMusicTopus(rows); case "json": default: return { diff --git a/src/middleware/Export.js b/src/middleware/Export.js new file mode 100644 index 0000000..380f025 --- /dev/null +++ b/src/middleware/Export.js @@ -0,0 +1,453 @@ +import momenttz from "moment-timezone"; +import xl from "excel4node"; + +class Export { + /** + * Méthode permettant de remplacer certains cartactères par leur équivalents html + * @param {String} str + * + * @return {String} + */ + static replaceSpecialChars(str) { + if (!str) { + return ""; + } + let final = str.toString(); + const find = ["&", "<", ">"]; + const replace = ["&", "<", ">"]; + + for (let i = 0; i < find.length; i += 1) { + final = final.replace(new RegExp(find[i], "g"), replace[i]); + } + + return final; + } + + /** + * Méthode permettant de convertir les rows en csv + * @param {Array} rows + * + * @return {string} + */ + static async convertToCsv(rows) { + let data = + "Artiste;Titre;Genre;Styles;Pays;Année;Date de sortie;Format\n\r"; + + for (let i = 0; i < rows.length; i += 1) { + const { + artists_sort, + title, + genres, + styles, + country, + year, + released, + formats, + } = rows[i]; + + let format = ""; + for (let j = 0; j < formats.length; j += 1) { + format += `${format !== "" ? ", " : ""}${formats[j].name}`; + } + + data += `${artists_sort};${title};${genres.join()};${styles.join()};${country};${year};${released};${format}\n\r`; + } + + return data; + } + + /** + * Méthode permettant de convertir les rows en fichier xls + * @param {Array} rows + * + * @return {Object} + */ + static async convertToXls(rows) { + const wb = new xl.Workbook(); + const ws = wb.addWorksheet("MusicTopus"); + + const headerStyle = wb.createStyle({ + font: { + color: "#FFFFFF", + size: 11, + }, + fill: { + type: "pattern", + patternType: "solid", + bgColor: "#595959", + fgColor: "#595959", + }, + }); + const style = wb.createStyle({ + font: { + color: "#000000", + size: 11, + }, + numberFormat: "0000", + }); + + const header = [ + "Artiste", + "Titre", + "Genre", + "Styles", + "Pays", + "Année", + "Date de sortie", + "Format", + ]; + for (let i = 0; i < header.length; i += 1) { + ws.cell(1, i + 1) + .string(header[i]) + .style(headerStyle); + } + + for (let i = 0; i < rows.length; i += 1) { + const currentRow = i + 2; + const { + artists_sort, + title, + genres, + styles, + country, + year, + released, + formats, + } = rows[i]; + + let format = ""; + for (let j = 0; j < formats.length; j += 1) { + format += `${format !== "" ? ", " : ""}${formats[j].name}`; + } + + ws.cell(currentRow, 1).string(artists_sort).style(style); + ws.cell(currentRow, 2).string(title).style(style); + ws.cell(currentRow, 3).string(genres.join()).style(style); + ws.cell(currentRow, 4).string(styles.join()).style(style); + if (country) { + ws.cell(currentRow, 5).string(country).style(style); + } + if (year) { + ws.cell(currentRow, 6).number(year).style(style); + } + if (released) { + ws.cell(currentRow, 7) + .date(momenttz.tz(released, "Europe/Paris").hour(12)) + .style({ numberFormat: "dd/mm/yyyy" }); + } + ws.cell(currentRow, 8).string(format).style(style); + } + + return wb; + } + + /** + * Méthode permettant de convertir les rows en csv pour importer dans MusicTopus + * @param {Array} rows + * + * @return {string} + */ + static async convertToXml(rows) { + let data = '\n\r'; + + for (let i = 0; i < rows.length; i += 1) { + const { + discogsId, + year, + released, + uri, + artists, + artists_sort, + labels, + series, + companies, + formats, + title, + country, + notes, + identifiers, + videos, + genres, + styles, + tracklist, + extraartists, + images, + thumb, + } = rows[i]; + + let artistsList = ""; + let labelList = ""; + let serieList = ""; + let companiesList = ""; + let formatsList = ""; + let identifiersList = ""; + let videosList = ""; + let genresList = ""; + let stylesList = ""; + let tracklistList = ""; + let extraartistsList = ""; + let imagesList = ""; + + for (let j = 0; j < artists.length; j += 1) { + artistsList += ` + ${Export.replaceSpecialChars(artists[j].name)} + ${Export.replaceSpecialChars(artists[j].anv)} + ${Export.replaceSpecialChars(artists[j].join)} + ${Export.replaceSpecialChars(artists[j].role)} + ${Export.replaceSpecialChars(artists[j].tracks)} + ${Export.replaceSpecialChars(artists[j].id)} + ${Export.replaceSpecialChars( + artists[j].resource_url + )} + ${Export.replaceSpecialChars( + artists[j].thumbnail_url + )} + `; + } + + for (let j = 0; j < labels.length; j += 1) { + labelList += ` + `; + } + + for (let j = 0; j < series.length; j += 1) { + serieList += ` + ${Export.replaceSpecialChars(series[j].name)} + ${Export.replaceSpecialChars(series[j].catno)} + ${Export.replaceSpecialChars( + series[j].entity_type + )} + ${Export.replaceSpecialChars( + series[j].entity_type_name + )} + ${Export.replaceSpecialChars(series[j].id)} + ${Export.replaceSpecialChars( + series[j].resource_url + )} + ${Export.replaceSpecialChars( + series[j].thumbnail_url + )} + + `; + } + + for (let j = 0; j < companies.length; j += 1) { + companiesList += ` + ${Export.replaceSpecialChars(companies[j].name)} + ${Export.replaceSpecialChars(companies[j].catno)} + ${Export.replaceSpecialChars( + companies[j].entity_type + )} + ${Export.replaceSpecialChars( + companies[j].entity_type_name + )} + ${Export.replaceSpecialChars(companies[j].id)} + ${Export.replaceSpecialChars( + companies[j].resource_url + )} + ${Export.replaceSpecialChars( + companies[j].thumbnail_url + )} + + `; + } + + for (let j = 0; j < formats.length; j += 1) { + let descriptions = ""; + if (formats[j].descriptions) { + for ( + let k = 0; + k < formats[j].descriptions.length; + k += 1 + ) { + descriptions += `${formats[j].descriptions[k]} + `; + } + } + formatsList += ` + ${Export.replaceSpecialChars(formats[j].name)} + ${Export.replaceSpecialChars(formats[j].qty)} + ${Export.replaceSpecialChars(formats[j].text)} + + ${descriptions} + + + `; + } + + for (let j = 0; j < identifiers.length; j += 1) { + identifiersList += ` + ${Export.replaceSpecialChars(identifiers[j].type)} + ${Export.replaceSpecialChars(identifiers[j].value)} + ${Export.replaceSpecialChars( + identifiers[j].description + )} + + `; + } + + for (let j = 0; j < videos.length; j += 1) { + videosList += ` + `; + } + + for (let j = 0; j < genres.length; j += 1) { + genresList += `${Export.replaceSpecialChars( + genres[j] + )} + `; + } + + for (let j = 0; j < styles.length; j += 1) { + stylesList += ` + `; + } + + for (let j = 0; j < tracklist.length; j += 1) { + tracklistList += ` + ${Export.replaceSpecialChars(tracklist[j].title)} + + `; + } + + for (let j = 0; j < extraartists.length; j += 1) { + extraartistsList += ` + ${Export.replaceSpecialChars(extraartists[j].name)} + ${Export.replaceSpecialChars(extraartists[j].anv)} + ${Export.replaceSpecialChars(extraartists[j].join)} + ${Export.replaceSpecialChars(extraartists[j].role)} + ${Export.replaceSpecialChars( + extraartists[j].tracks + )} + ${Export.replaceSpecialChars(extraartists[j].id)} + ${Export.replaceSpecialChars( + extraartists[j].resource_url + )} + ${Export.replaceSpecialChars( + extraartists[j].thumbnail_url + )} + + `; + } + + for (let j = 0; j < images.length; j += 1) { + imagesList += ` + ${Export.replaceSpecialChars(images[j].uri)} + ${Export.replaceSpecialChars( + images[j].resource_url + )} + ${Export.replaceSpecialChars( + images[j].resource_url + )} + + `; + } + + data += ` + + ${discogsId} + ${Export.replaceSpecialChars(title)} + ${Export.replaceSpecialChars(artists_sort)} + + ${artistsList} + + ${year} + ${Export.replaceSpecialChars(country)} + ${released} + ${uri} + ${thumb} + + ${labelList} + + + ${serieList} + + + ${companiesList} + + + ${formatsList} + + ${Export.replaceSpecialChars(notes)} + + ${identifiersList} + + + ${videosList} + + + ${genresList} + + + ${stylesList} + + + ${tracklistList} + + + ${extraartistsList} + + + ${imagesList} + +`; + } + + return `${data}`; + } + + /** + * Méthode permettant de convertir les rows en csv pour importer dans MusicTopus + * @param {Array} rows + * + * @return {string} + */ + static async convertToMusicTopus(rows) { + let data = "itemId;createdAt;updatedAt\n\r"; + + for (let i = 0; i < rows.length; i += 1) { + const { discogsId, createdAt, updatedAt } = rows[i]; + + data += `${discogsId};${createdAt};${updatedAt}\n\r`; + } + + data += "v1.0"; + + return data; + } +} + +export default Export; diff --git a/src/middleware/Jobs.js b/src/middleware/Jobs.js new file mode 100644 index 0000000..ccb2cf9 --- /dev/null +++ b/src/middleware/Jobs.js @@ -0,0 +1,128 @@ +/* eslint-disable no-await-in-loop */ +import ErrorEvent from "../libs/error"; +import { uploadFromUrl } from "../libs/aws"; +import { getAlbumDetails } from "../helpers"; + +import JobsModel from "../models/jobs"; +import AlbumsModel from "../models/albums"; + +class Jobs { + /** + * Méthode permettant de télécharger toute les images d'un album + * @param {ObjectId} itemId + */ + static async importAlbumAssets(itemId) { + const album = await AlbumsModel.findById(itemId); + + if (!album) { + throw new ErrorEvent( + 404, + "Item non trouvé", + `L'album avant l'id ${itemId} n'existe plus dans la collection` + ); + } + + const item = await getAlbumDetails(album.discogsId); + + if (!item) { + throw new ErrorEvent( + 404, + "Erreur de communication", + "Erreur lors de la récupération des informations sur Discogs" + ); + } + + if (item.thumb) { + album.thumb = await uploadFromUrl(item.thumb); + album.thumbType = "local"; + } + const { images } = item; + if (images && images.length > 0) { + for (let i = 0; i < images.length; i += 1) { + images[i].uri150 = await uploadFromUrl(images[i].uri150); + images[i].uri = await uploadFromUrl(images[i].uri); + } + } + + album.images = images; + + await album.save(); + + return true; + } + + /** + * Point d'entrée + * @param {String} state + * + * @return {Object} + */ + async run(state = "NEW") { + const job = await JobsModel.findOne({ + state, + tries: { + $lte: 5, + }, + }); + + if (!job) { + return { message: "All jobs done" }; + } + + job.state = "IN-PROGRESS"; + + await job.save(); + + try { + switch (job.model) { + case "Albums": + await Jobs.importAlbumAssets(job.id); + break; + default: + throw new ErrorEvent( + 500, + "Job inconnu", + `Le job avec l'id ${job._id} n'est pas un job valide` + ); + } + + job.state = "SUCCESS"; + + await job.save(); + + return this.run(state); + } catch (err) { + job.state = "ERROR"; + job.lastTry = new Date(); + job.lastErrorMessage = err.message; + job.tries += 1; + + await job.save(); + + throw err; + } + } + + /** + * Méthode permettant de créer tous les jobs + * + * @return {Object} + */ + static async populate() { + const albums = await AlbumsModel.find(); + + for (let i = 0; i < albums.length; i += 1) { + const jobData = { + model: "Albums", + id: albums[i]._id, + }; + + const job = new JobsModel(jobData); + await job.save(); + } + + return { message: `${albums.length} jobs ajouté à la file d'attente` }; + } +} + +export default Jobs; diff --git a/src/models/albums.js b/src/models/albums.js index 0180dad..e08b7ad 100644 --- a/src/models/albums.js +++ b/src/models/albums.js @@ -29,6 +29,7 @@ const AlbumSchema = new mongoose.Schema( extraartists: Array, images: Array, thumb: String, + thumbType: String, }, { timestamps: true } ); diff --git a/src/models/jobs.js b/src/models/jobs.js new file mode 100644 index 0000000..ba3727d --- /dev/null +++ b/src/models/jobs.js @@ -0,0 +1,24 @@ +import mongoose from "mongoose"; + +const { Schema } = mongoose; + +const JobSchema = new mongoose.Schema( + { + model: String, + id: Schema.Types.ObjectId, + state: { + type: String, + enum: ["NEW", "IN-PROGRESS", "ERROR", "SUCCESS"], + default: "NEW", + }, + lastTry: Date, + lastErrorMessage: String, + tries: { + type: Number, + default: 0, + }, + }, + { timestamps: true } +); + +export default mongoose.model("Jobs", JobSchema); diff --git a/src/routes/jobs.js b/src/routes/jobs.js new file mode 100644 index 0000000..6bac899 --- /dev/null +++ b/src/routes/jobs.js @@ -0,0 +1,40 @@ +import express from "express"; +import passport from "passport"; + +import Jobs from "../middleware/Jobs"; + +// eslint-disable-next-line new-cap +const router = express.Router(); + +router.route("/").get( + passport.authenticate(["jobs"], { + session: false, + }), + async (req, res, next) => { + try { + const job = new Jobs(); + const data = await job.run(req.query.state); + + return res.status(200).json(data).end(); + } catch (err) { + return next(err); + } + } +); + +router.route("/populate").get( + passport.authenticate(["jobs"], { + session: false, + }), + async (req, res, next) => { + try { + const data = await Jobs.populate(); + + return res.status(200).json(data).end(); + } catch (err) { + return next(err); + } + } +); + +export default router; diff --git a/views/pages/mon-compte/ma-collection/details.ejs b/views/pages/mon-compte/ma-collection/details.ejs index 12490c0..bd97f90 100644 --- a/views/pages/mon-compte/ma-collection/details.ejs +++ b/views/pages/mon-compte/ma-collection/details.ejs @@ -170,7 +170,7 @@ } }, created() { - this.setTrackList(); + this.setTrackList(); this.setIdentifiers(); window.addEventListener("keydown", this.changeImage); @@ -231,7 +231,7 @@ }, showGallery(event) { const item = event.target.tagName === 'IMG' ? event.target.parentElement : event.target; - + const { index, } = item.dataset;