cangui/manga-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first comit

Browse Source
This commit is contained in:
cangui 2024-10-05 10:07:31 +02:00
commit e3ef711383
21 changed files with 2029 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.dockerignore Normal file
View File

@ -0,0 +1,3 @@
node_modules
npm-debug.log
.env

5
.env Normal file
View File

@ -0,0 +1,5 @@
DB_NAME=manga_database
DB_USER=root
DB_PASSWORD=secret
DB_HOST=localhost
PORT=8080

35
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,35 @@
stages:
- deploy
deploy_to_portainer:
stage: deploy
image:
name: curlimages/curl:latest
entrypoint: [""]
script:
- echo "Installation de jq"
- curl -L -o /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
- chmod +x /usr/bin/jq
- echo "Début du déploiement vers Portainer"
- echo "Téléchargement du fichier docker-compose.yml depuis le dépôt"
- 'curl -s -H "PRIVATE-TOKEN: $CI_JOB_TOKEN" -o docker-compose.yml "$CI_PROJECT_URL/raw/$CI_COMMIT_REF_NAME/docker-compose.yml"'
- echo "Mise à jour ou création de la stack dans Portainer"
- |
STACK_ID=$(curl -s -H "Authorization: Bearer $PORTAINER_TOKEN" "$PORTAINER_URL/stacks?filters=%7B%22Name%22%3A%5B%22$STACK_NAME%22%5D%7D" | jq -r '.[0].Id')
if [ "$STACK_ID" = "null" ] || [ -z "$STACK_ID" ]; then
echo "La stack n'existe pas, création en cours"
STACK_CREATE=$(curl -s -X POST "$PORTAINER_URL/stacks?type=1&method=string&endpointId=$ENDPOINT_ID" \
-H "Authorization: Bearer $PORTAINER_TOKEN" \
-H "Content-Type: application/json" \
-d "{\"Name\": \"$STACK_NAME\", \"StackFileContent\": \"$(cat docker-compose.yml | sed 's/"/\\"/g' | sed 's/$/\\n/' | tr -d '\n')\", \"Prune\": false}")
echo "Réponse de création : $STACK_CREATE"
else
echo "La stack existe, mise à jour en cours"
STACK_UPDATE=$(curl -s -X PUT "$PORTAINER_URL/stacks/$STACK_ID?endpointId=$ENDPOINT_ID" \
-H "Authorization: Bearer $PORTAINER_TOKEN" \
-H "Content-Type: application/json" \
-d "{\"StackFileContent\": \"$(cat docker-compose.yml | sed 's/"/\\"/g' | sed 's/$/\\n/' | tr -d '\n')\", \"Prune\": false, \"PullImage\": true}")
echo "Réponse de mise à jour : $STACK_UPDATE"
fi
only:
- main

8
.idea/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

8
.idea/dev manag.iml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

8
.idea/modules.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/dev manag.iml" filepath="$PROJECT_DIR$/.idea/dev manag.iml" />
</modules>
</component>
</project>

19
.idea/php.xml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MessDetectorOptionsConfiguration">
<option name="transferred" value="true" />
</component>
<component name="PHPCSFixerOptionsConfiguration">
<option name="transferred" value="true" />
</component>
<component name="PHPCodeSnifferOptionsConfiguration">
<option name="highlightLevel" value="WARNING" />
<option name="transferred" value="true" />
</component>
<component name="PhpStanOptionsConfiguration">
<option name="transferred" value="true" />
</component>
<component name="PsalmOptionsConfiguration">
<option name="transferred" value="true" />
</component>
</project>

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

20
Dockerfile Normal file
View File

@ -0,0 +1,20 @@
# Utiliser une image Node.js officielle comme image de base
FROM node:18
# Créer un répertoire de travail dans le conteneur
WORKDIR /app
# Copier les fichiers package.json et package-lock.json
COPY package*.json ./
# Installer les dépendances
RUN npm install
# Copier le reste du code de l'application
COPY . .
# Exposer le port sur lequel votre application s'exécute (assurez-vous que c'est le bon)
EXPOSE 8080
# Démarrer l'application
CMD [ "node", "app.js" ]

20
app.js Normal file
View File

@ -0,0 +1,20 @@
const express = require('express');
const sequelize = require('./config');
const Manga = require('./models/manga');
const Chapter = require('./models/chapter');
const mangaRoutes = require('./routes/mangaRoutes');
require('dotenv').config();
const app = express();
const PORT = process.env.PORT || 8080;
app.use(express.json());
app.use('/', mangaRoutes);
sequelize.sync().then(() => {
app.listen(PORT, () => {
console.log(`Serveur en écoute sur le port ${PORT}`);
});
}).catch((err) => {
console.error('Erreur lors de la synchronisation avec la base de données:', err);
});

16
config.js Normal file
View File

@ -0,0 +1,16 @@
require('dotenv').config();
const { Sequelize } = require('sequelize');
const sequelize = new Sequelize(
process.env.DB_NAME || 'manga_database',
process.env.DB_USER || 'root',
process.env.DB_PASSWORD || 'secret',
{
host: process.env.DB_HOST || 'localhost',
dialect: 'mysql', // ou 'postgres', selon votre choix
logging: false,
}
);
module.exports = sequelize;

177
connectors.json Normal file
View File

@ -0,0 +1,177 @@
[
{
"id": "1",
"baseUrl": "https://www.lelmanga.com",
"url": "https://www.lelmanga.com/manga",
"name": "LelManga P4",
"titleSelector": "div.tt",
"imageMangaSelector": ".limit img",
"attributManga": "src",
"categorieElement": ".info-desc",
"categorieAtribute": "a",
"categories": ".mgen",
"rate": ".rating-prc .num",
"description": ".entry-content-single p",
"paramSup": "?page=",
"chapterPage": ".bsx a",
"chapterSelector": "li[data-num]",
"chapterNumSelector": "span.chapternum",
"chapterTitleSelector": "a",
"chapterDateSelector": "span.chapterdate",
"imageSelector": "#readerarea noscript",
"imageAttribute": "img",
"navigatePageImage": false,
"status":"Functional",
"localisation": "fr"
},
{
"id": "2",
"baseUrl": "https://scan-trad.com",
"url": "https://scan-trad.com/manga",
"name": "scan-trad.com P16",
"titleSelector": ".series-name",
"imageMangaSelector": ".series img",
"attributManga": "data-src",
"categorieElement": ".card-series-detail .col-6.col-md-12.mb-4",
"categorieAtribute": "a",
"categories": ".badge.bg-light.text-dark",
"rate": ".rate-value span",
"description": ".col-12 p",
"paramSup": "?page=",
"chapterPage": ".link-series",
"chapterSelector": ".col-chapter",
"chapterNumSelector": ".mb-0",
"chapterTitleSelector": "a",
"chapterDateSelector": "div.text-muted",
"imageSelector": ".book-page",
"imageAttribute": "img",
"navigatePageImage": false,
"status":"Functional",
"localisation": "fr"
},
{
"id": "4",
"baseUrl": "https://astral-manga.fr",
"url": "https://astral-manga.fr/manga/",
"name": "astral-manga P14",
"titleSelector": ".manga .h5 a",
"imageMangaSelector": ".manga img",
"attributManga": "data-src",
"categorieElement": ".genres-content",
"categorieAtribute": "a",
"categories": ".genres-content",
"rate": ".allow_vote .total_votes",
"description": ".manga-excerpt p",
"paramSup": "page/",
"chapterPage": ".manga .h5 a",
"chapterSelector": ".listing-chapters_wrap ul li",
"chapterNumSelector": ".wp-manga-chapter a",
"chapterTitleSelector": "a",
"chapterDateSelector": "span.chapter-release-date",
"imageSelector": ".reading-content",
"imageAttribute": "img.wp-manga-chapter-img",
"navigatePageImage": true,
"status":"Functional",
"localisation": "fr"
},
{
"id": "5",
"baseUrl": "https://bato.to",
"url": "https://bato.to/browse?genres=|gore,bloody,violence,ecchi,adult,mature,smut,hentai&langs=en&sort=update.za",
"name": "bato.to P1184",
"titleSelector": ".item-title",
"imageMangaSelector": ".item-cover img",
"attributManga": "src",
"categorieElement": ".attr-item span span",
"categorieAtribute": "span",
"categories": ".attr-item span span",
"rate": ".attr-main .attr-item span",
"description": ".limit-html",
"paramSup": "&page=",
"chapterPage": ".item-cover",
"chapterSelector": ".item",
"chapterNumSelector": ".item a b",
"chapterTitleSelector": ".chapt",
"chapterDateSelector": ".item a b",
"imageSelector": "#viewer",
"imageAttribute": "img",
"navigatePageImage": false,
"status": "Functional",
"localisation": "us"
},
{
"id": "6",
"baseUrl": "https://mangahub.io",
"url": "https://mangahub.io/search/",
"name": "mangahub.io [limit for moment preview max 6 img] P1946",
"titleSelector": ".media .media-body h4 a",
"imageMangaSelector": ".media .media-left img",
"attributManga": "src",
"categorieElement": "._3Czbn",
"categorieAtribute": "a",
"categories": "._3Czbn",
"rate": "._3SlhO",
"description": "h1 small",
"paramSup": "page/",
"chapterPage": ".media .media-left a",
"chapterSelector": "ul li.list-group-item",
"chapterNumSelector": "ul li.list-group-item ._2IG5P",
"chapterTitleSelector": "ul li.list-group-item ._3pfyN",
"chapterDateSelector": "ul li.list-group-item small",
"imageSelector": "._2aWyJ",
"imageAttribute": "img",
"navigatePageImage": false,
"status": "Functional",
"localisation": "us"
},
{
"id": "7",
"baseUrl": "https://bato.to",
"url": "https://bato.to/browse?genres=|gore,bloody,violence,ecchi,adult,mature,smut,hentai&langs=fr&sort=title.az",
"name": "bato.to P36",
"titleSelector": ".item-title",
"imageMangaSelector": ".item-cover img",
"attributManga": "src",
"categorieElement": ".attr-item span span",
"categorieAtribute": "span",
"categories": ".attr-item span span",
"rate": ".attr-main .attr-item span",
"description": ".limit-html",
"paramSup": "&page=",
"chapterPage": ".item-cover",
"chapterSelector": ".item",
"chapterNumSelector": ".item a b",
"chapterTitleSelector": ".chapt",
"chapterDateSelector": ".item a b",
"imageSelector": "#viewer",
"imageAttribute": "img",
"navigatePageImage": false,
"status": "Functional",
"localisation": "fr"
},{
"id": "8",
"baseUrl": "https://toonclash.com",
"url": "https://toonclash.com/manga/",
"name": "toonclash.com P256",
"titleSelector": ".post-title .h5",
"imageMangaSelector": ".manga .c-image-hover img",
"attributManga": "data-src",
"categorieElement": ".genres-content",
"categorieAtribute": "a",
"categories": ".genres-content",
"rate": "#averagerate",
"description": ".description-summary .summary__content",
"paramSup": "page/",
"chapterPage": ".post-title .h5 a",
"chapterSelector": ".listing-chapters_wrap ul li",
"chapterNumSelector": ".listing-chapters_wrap ul li a",
"chapterTitleSelector": ".listing-chapters_wrap ul li a",
"chapterDateSelector": ".listing-chapters_wrap ul li a",
"imageSelector": "div.reading-content",
"imageAttribute": "img",
"navigatePageImage": false,
"status": "Functional",
"localisation": "us"
}
]

153
controllers/mangaController.js Normal file
View File

@ -0,0 +1,153 @@
const axios = require('axios');
const cheerio = require('cheerio');
const Manga = require('../models/manga');
const Chapter = require('../models/chapter');
const connectors = require('../connectors.json');
class MangaController {
constructor() {}
async fetchAndStoreMangas(connectorId, localisation) {
const connectorsToFetch = connectors.filter((config) => {
const matchesConnectorId = !connectorId || config.id === connectorId;
const matchesLocalisation = !localisation || config.localisation === localisation;
return matchesConnectorId && matchesLocalisation;
});
for (const config of connectorsToFetch) {
await this.fetchMangas(config);
}
}
async fetchMangas(config) {
let page = 1;
let shouldContinue = true;
while (shouldContinue) {
const pageUrl = config.paramSup !== "null" ? `${config.url}${config.paramSup}${page}` : config.url;
shouldContinue = config.paramSup !== "null";
try {
const response = await axios.get(pageUrl);
const $ = cheerio.load(response.data);
const titleElements = $(config.titleSelector);
const imageElements = $(config.imageMangaSelector);
const linkElements = $(config.chapterPage);
if (!titleElements.length || !imageElements.length || !linkElements.length) break;
const mangasOnPage = [];
for (let i = 0; i < titleElements.length; i++) {
const title = $(titleElements[i]).text().trim();
const imageUrl = $(imageElements[i]).attr(config.attributManga) || '';
let linkChapter = $(linkElements[i]).attr('href') || '';
if (linkChapter.startsWith('/')) linkChapter = config.baseUrl + linkChapter;
if (!title) continue;
try {
const mangaPageResponse = await axios.get(linkChapter);
const $$ = cheerio.load(mangaPageResponse.data);
const description = $$(config.description).text().trim() || 'Not available';
const categories = [];
const categorieElement = $$(config.categorieElement);
const categoriesSelector = config.categorieElement === config.categories
? $$(config.categorieElement)
: categorieElement.find(config.categories);
categoriesSelector.each((_, elem) => {
categories.push($$(elem).text().trim());
});
const rate = $$(config.rate).text().trim() || '0';
const chapters = await this.fetchChapters(linkChapter, config);
// Enregistrer le manga dans la base de données
const [manga, created] = await Manga.findOrCreate({
where: { title },
defaults: {
connectorId: config.id,
connectorName: config.name,
imageUrl,
linkChapter,
description,
categories,
rate,
},
});
if (!created) {
// Mettre à jour le manga existant
await manga.update({
connectorId: config.id,
connectorName: config.name,
imageUrl,
linkChapter,
description,
categories,
rate,
});
}
// Enregistrer les chapitres
for (const chapterData of chapters) {
await Chapter.findOrCreate({
where: { chapterLink: chapterData.chapterLink },
defaults: {
...chapterData,
mangaId: manga.id,
},
});
}
} catch (e) {
console.error(`Erreur lors du traitement du manga ${title}: ${e.message}`);
continue;
}
}
page++;
} catch (e) {
console.error(`Erreur lors de la récupération de la page ${pageUrl}: ${e.message}`);
shouldContinue = false;
}
}
}
async fetchChapters(linkChapter, config) {
try {
const response = await axios.get(linkChapter);
const $ = cheerio.load(response.data);
const chapterElements = $(config.chapterSelector);
const chapters = [];
chapterElements.each((_, elem) => {
const chapterNum = $(elem).find(config.chapterNumSelector).text().trim() || 'N/A';
let chapterLink = $(elem).find(config.chapterTitleSelector).attr('href') || '';
const chapterTitle = $(elem).find(config.chapterTitleSelector).text().replace(/^\s+|\s+$/g, '') || 'N/A';
const chapterDate = $(elem).find(config.chapterDateSelector).text().trim() || '';
if (chapterLink.startsWith('/')) chapterLink = config.baseUrl + chapterLink;
console.log(chapterTitle)
chapters.push({
chapterNum,
chapterTitle,
chapterLink,
chapterDate,
});
});
return chapters;
} catch (e) {
console.error(`Erreur lors de la récupération des chapitres: ${e.message}`);
return [];
}
}
}
module.exports = new MangaController();

41
docker-compose.yml Normal file
View File

@ -0,0 +1,41 @@
version: '3.8'
services:
app:
build: .
ports:
- '8080:8080'
environment:
- DB_HOST=db
- DB_USER=root
- DB_PASSWORD=secret
- DB_NAME=manga_database
- PORT=8080
depends_on:
- db
volumes:
- .:/app
- /app/node_modules
db:
image: mysql:8.0
environment:
- MYSQL_ROOT_PASSWORD=secret
- MYSQL_DATABASE=manga_database
volumes:
- db_data:/var/lib/mysql
ports:
- '3306:3306'
phpmyadmin:
image: phpmyadmin/phpmyadmin
restart: always
ports:
- '8081:80'
environment:
- PMA_HOST=db
- PMA_USER=root
- PMA_PASSWORD=secret
depends_on:
- db
volumes:
db_data:

23
models/chapter.js Normal file
View File

@ -0,0 +1,23 @@
const { DataTypes } = require('sequelize');
const sequelize = require('../config');
const Chapter = sequelize.define('Chapter', {
chapterNum: {
type: DataTypes.TEXT,
allowNull: false,
},
chapterTitle: {
type: DataTypes.TEXT,
allowNull: false,
},
chapterLink: {
type: DataTypes.TEXT,
allowNull: false,
},
chapterDate: {
type: DataTypes.TEXT,
allowNull: true,
},
});
module.exports = Chapter;

0
models/connectorConfig.js Normal file
View File

47
models/manga.js Normal file
View File

@ -0,0 +1,47 @@
const { DataTypes } = require('sequelize');
const sequelize = require('../config');
const Chapter = require('./chapter');
const Manga = sequelize.define('Manga', {
connectorId: {
type: DataTypes.STRING,
allowNull: false,
},
connectorName: {
type: DataTypes.STRING,
allowNull: false,
},
title: {
type: DataTypes.TEXT,
allowNull: false,
},
imageUrl: {
type: DataTypes.TEXT,
allowNull: false,
},
linkChapter: {
type: DataTypes.TEXT,
allowNull: false,
},
isFavorite: {
type: DataTypes.BOOLEAN,
defaultValue: false,
},
description: {
type: DataTypes.TEXT,
allowNull: false,
},
categories: {
type: DataTypes.JSON, // Stocke la liste des catégories
allowNull: false,
},
rate: {
type: DataTypes.TEXT,
defaultValue: '0',
},
});
Manga.hasMany(Chapter, { as: 'chapters', foreignKey: 'mangaId' });
Chapter.belongsTo(Manga, { foreignKey: 'mangaId' });
module.exports = Manga;

2
note.txt Normal file
View File

@ -0,0 +1,2 @@
docker-compose build
docker-compose up

1359
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

19
package.json Normal file
View File

@ -0,0 +1,19 @@
{
"name": "manga_server",
"version": "1.0.0",
"main": "app.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"description": "",
"dependencies": {
"axios": "^1.7.7",
"cheerio": "^1.0.0",
"dotenv": "^16.4.5",
"express": "^4.21.0",
"mysql2": "^3.11.3",
"sequelize": "^6.37.3"
}
}

60
routes/mangaRoutes.js Normal file
View File

@ -0,0 +1,60 @@
const express = require('express');
const router = express.Router();
const Manga = require('../models/manga');
const Chapter = require('../models/chapter');
const mangaController = require('../controllers/mangaController');
const connectors = require('../connectors.json');
router.get('/fetch', async (req, res) => {
const connectorId = req.query.connectorId;
const localisation = req.query.localisation;
await mangaController.fetchAndStoreMangas(connectorId, localisation);
res.send('Données récupérées avec succès');
});
router.get('/mangas', async (req, res) => {
const connectorId = req.query.connectorId;
const localisation = req.query.localisation;
let whereClause = {};
if (connectorId) {
whereClause.connectorId = connectorId;
}
if (localisation) {
const connectorsForLoc = connectors.filter((config) => config.localisation === localisation);
const connectorNames = connectorsForLoc.map((c) => c.name);
if (connectorNames.length > 0) {
whereClause.connectorName = connectorNames;
} else {
return res.json([]);
}
}
const mangas = await Manga.findAll({
where: whereClause,
include: [{ model: Chapter, as: 'chapters' }],
});
res.json(mangas);
});
router.get('/mangas/:title', async (req, res) => {
const title = req.params.title;
const manga = await Manga.findOne({
where: { title },
include: [{ model: Chapter, as: 'chapters' }],
});
if (!manga) {
return res.status(404).send('Manga non trouvé');
}
res.json(manga);
});
module.exports = router;