347 lines
8.8 KiB
JavaScript
347 lines
8.8 KiB
JavaScript
/*
|
|
* We got so many locals that building them all in one go can lead to out-of-memory error
|
|
* Lets split that here
|
|
*/
|
|
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
const readline = require('readline');
|
|
const { spawn } = require('child_process');
|
|
const webpack = require('webpack');
|
|
const validate = require("ttag-cli/dist/src/commands/validate").default;
|
|
|
|
const minifyCss = require('./minifyCss');
|
|
const serverConfig = require('../webpack.config.server.js');
|
|
const clientConfig = require('../webpack.config.client.js');
|
|
|
|
let langs = 'all';
|
|
let doBuildServer = false;
|
|
let doBuildClient = false;
|
|
let parallel = false;
|
|
let recursion = false;
|
|
let onlyValidate = false;
|
|
for (let i = 0; i < process.argv.length; i += 1) {
|
|
switch (process.argv[i]) {
|
|
case '--langs': {
|
|
const newLangs = process.argv[++i];
|
|
if (newLangs) langs = newLangs;
|
|
break;
|
|
}
|
|
case '--client':
|
|
doBuildClient = true;
|
|
break;
|
|
case `--server`:
|
|
doBuildServer = true;
|
|
break;
|
|
case '--parallel':
|
|
parallel = true;
|
|
break;
|
|
case '--recursion':
|
|
recursion = true;
|
|
break;
|
|
case '--validate':
|
|
onlyValidate = true;
|
|
break;
|
|
default:
|
|
// nothing
|
|
}
|
|
}
|
|
if (!doBuildServer && !doBuildClient) {
|
|
doBuildServer = true;
|
|
doBuildClient = true;
|
|
}
|
|
|
|
/*
|
|
* get available locals based on the files available in ../i18n
|
|
*/
|
|
function getAllAvailableLocals() {
|
|
const langDir = path.resolve(__dirname, '..', 'i18n');
|
|
const langs = fs.readdirSync(langDir)
|
|
.filter((e) => (e.endsWith('.po') && !e.startsWith('ssr')))
|
|
.map((l) => l.slice(0, -3));
|
|
langs.unshift('en');
|
|
return langs;
|
|
}
|
|
|
|
/*
|
|
* get amount of msgid and msgstr of po file
|
|
*/
|
|
function getPoFileStats(file) {
|
|
return new Promise((resolve) => {
|
|
const fileStream = fs.createReadStream(file);
|
|
const lineReader = readline.createInterface({
|
|
input: fileStream,
|
|
crlfDelay: Infinity,
|
|
});
|
|
|
|
let msgid = 0;
|
|
let msgstr = 0;
|
|
|
|
lineReader.on('line', (l) => {
|
|
l = l.trim();
|
|
if (l.endsWith('""')) {
|
|
return;
|
|
}
|
|
let seperator = l.indexOf(' ');
|
|
if (seperator === -1) {
|
|
seperator = l.indexOf('\t');
|
|
}
|
|
if (seperator === -1) {
|
|
return;
|
|
}
|
|
const tag = l.substring(0, seperator);
|
|
if (tag === 'msgid') {
|
|
msgid += 1;
|
|
} else if (tag === 'msgstr') {
|
|
msgstr += 1;
|
|
}
|
|
});
|
|
|
|
lineReader.on('close', (l) => {
|
|
resolve({ msgid, msgstr });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function filterLackingLocals(langs, percentage) {
|
|
langs = langs.filter((l) => l !== 'en');
|
|
const promises = [];
|
|
const { msgid, msgstr } = await getPoFileStats(path.resolve(
|
|
__dirname, '..', 'i18n', `template.pot`,
|
|
));
|
|
|
|
const langStats = await Promise.all(langs
|
|
.map((l) => getPoFileStats(
|
|
path.resolve(__dirname, '..', 'i18n', `${l}.po`),
|
|
)));
|
|
const goodLangs = [ 'en' ];
|
|
const badLangs = [];
|
|
for (let i = 0; i < langs.length; i += 1) {
|
|
const lang = langs[i];
|
|
const stats = langStats[i];
|
|
const percent = Math.floor(stats.msgstr / msgid * 100);
|
|
if (percent >= percentage) {
|
|
goodLangs.push(lang);
|
|
} else {
|
|
console.log(`Lang ${lang} completion:`, percent, '%');
|
|
badLangs.push(lang);
|
|
}
|
|
}
|
|
return {
|
|
goodLangs,
|
|
badLangs,
|
|
};
|
|
}
|
|
|
|
/*
|
|
* check if language files contain errors
|
|
*/
|
|
function validateLangs(langs) {
|
|
console.log('Validating language files...');
|
|
const langDir = path.resolve(__dirname, '..', 'i18n');
|
|
const brokenLangs = [];
|
|
for (const lang of langs) {
|
|
const langFiles = [`${lang}.po`, `ssr-${lang}.po`];
|
|
for (const langFile of langFiles) {
|
|
process.stdout.clearLine(0);
|
|
process.stdout.cursorTo(0);
|
|
process.stdout.write(`i18n/${langFile} `);
|
|
filePath = path.join(langDir, langFile);
|
|
if (!fs.existsSync(filePath)) {
|
|
continue;
|
|
}
|
|
try {
|
|
validate(filePath);
|
|
} catch {
|
|
brokenLangs.push(langFile);
|
|
}
|
|
}
|
|
}
|
|
process.stdout.clearLine(0);
|
|
process.stdout.cursorTo(0);
|
|
return brokenLangs;
|
|
}
|
|
|
|
|
|
function compile(webpackConfig) {
|
|
return new Promise((resolve, reject) => {
|
|
webpack(webpackConfig, (err, stats) => {
|
|
if (err) {
|
|
return reject(err);
|
|
}
|
|
const statsConfig = (webpackConfig.length) ? webpackConfig[0].stats : webpackConfig.stats;
|
|
console.log(stats.toString(statsConfig))
|
|
return resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
function buildServer() {
|
|
console.log('-----------------------------');
|
|
console.log(`Build server...`);
|
|
console.log('-----------------------------');
|
|
const ts = Date.now();
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const argsc = (langs === 'all')
|
|
? ['webpack', '--env', 'extract', '--config', './webpack.config.server.js']
|
|
: ['webpack', '--config', './webpack.config.server.js']
|
|
const serverCompile = spawn('npx', argsc, {
|
|
shell: process.platform == 'win32',
|
|
});
|
|
serverCompile.stdout.on('data', (data) => {
|
|
console.log(data.toString());
|
|
});
|
|
serverCompile.stderr.on('data', (data) => {
|
|
console.error(data.toString());
|
|
});
|
|
serverCompile.on('close', (code) => {
|
|
if (code) {
|
|
reject(new Error('Server compilation failed!'));
|
|
} else {
|
|
console.log('---------------------------------------');
|
|
console.log(`Server Compilation finished in ${Math.floor((Date.now() - ts) / 1000)}s`);
|
|
console.log('---------------------------------------');
|
|
resolve();
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
function buildClients(slangs) {
|
|
return new Promise((resolve, reject) => {
|
|
const clientCompile = spawn('npm', ['run', 'build', '--', '--client', '--recursion', '--langs', slangs.join(',')], {
|
|
shell: process.platform == 'win32',
|
|
});
|
|
clientCompile.stdout.on('data', (data) => {
|
|
console.log(data.toString());
|
|
});
|
|
clientCompile.stderr.on('data', (data) => {
|
|
console.error(data.toString());
|
|
});
|
|
clientCompile.on('close', (code) => {
|
|
if (code) {
|
|
reject(new Error('Client compilation failed!'));
|
|
} else {
|
|
resolve();
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
async function buildClientsSync(avlangs) {
|
|
for(let i = 0; i < avlangs.length; i += 1) {
|
|
const lang = avlangs[i];
|
|
console.log(`Build client for locale ${lang}...`);
|
|
await compile(clientConfig({
|
|
development: false,
|
|
analyze: false,
|
|
extract: false,
|
|
locale: lang,
|
|
clean: false,
|
|
readonly: recursion,
|
|
}));
|
|
}
|
|
}
|
|
|
|
function buildClientsParallel(avlangs) {
|
|
const st = Date.now();
|
|
const numProc = 3;
|
|
let nump = Math.floor(avlangs.length / numProc);
|
|
if (!nump) nump = 1;
|
|
|
|
const promises = [];
|
|
while (avlangs.length >= nump) {
|
|
const slangs = avlangs.splice(0, nump);
|
|
promises.push(buildClients(slangs));
|
|
}
|
|
if (avlangs.length) {
|
|
promises.push(buildClientsSync(avlangs));
|
|
}
|
|
return Promise.all(promises);
|
|
}
|
|
|
|
async function build() {
|
|
const st = Date.now();
|
|
// cleanup old files
|
|
if (!recursion) {
|
|
fs.rmSync(path.resolve(__dirname, '..', 'node_modules', '.cache', 'webpack'), { recursive: true, force: true });
|
|
}
|
|
|
|
// decide which languages to build
|
|
let avlangs = getAllAvailableLocals();
|
|
if (langs !== 'all') {
|
|
avlangs = langs.split(',').map((l) => l.trim())
|
|
.filter((l) => avlangs.includes(l));
|
|
} else {
|
|
let badLangs;
|
|
({ goodLangs: avlangs, badLangs } = await filterLackingLocals(avlangs, 50));
|
|
if (badLangs.length) {
|
|
console.log(
|
|
'Skipping',
|
|
badLangs.length,
|
|
'locals because of low completion:',
|
|
badLangs,
|
|
);
|
|
}
|
|
}
|
|
if (!avlangs.length) {
|
|
console.error(`ERROR: language ${langs} not available`);
|
|
process.exit(1);
|
|
return;
|
|
}
|
|
console.log('Building', avlangs.length, 'locales:', avlangs);
|
|
|
|
const brokenLangs = validateLangs(avlangs);
|
|
if (brokenLangs.length) {
|
|
console.error('ERROR: Translation files', brokenLangs, 'contain errors.');
|
|
process.exit(2);
|
|
return;
|
|
}
|
|
if (onlyValidate) {
|
|
console.log('Validation complete, everything is fine.');
|
|
process.exit(0);
|
|
return;
|
|
}
|
|
|
|
const promises = [];
|
|
|
|
if (doBuildServer) {
|
|
promises.push(buildServer());
|
|
}
|
|
|
|
if (doBuildClient) {
|
|
if (!recursion) {
|
|
console.log('Building one client package...');
|
|
await compile(clientConfig({
|
|
development: false,
|
|
analyze: false,
|
|
extract: (langs === 'all'),
|
|
locale: avlangs.shift(),
|
|
clean: true,
|
|
readonly: false,
|
|
}));
|
|
|
|
console.log('-----------------------------');
|
|
console.log(`Minify CSS assets...`);
|
|
console.log('-----------------------------');
|
|
await minifyCss();
|
|
}
|
|
|
|
if (parallel) {
|
|
promises.push(buildClientsParallel(avlangs));
|
|
} else {
|
|
promises.push(buildClientsSync(avlangs));
|
|
}
|
|
}
|
|
await Promise.all(promises);
|
|
|
|
if (!recursion) {
|
|
console.log(`Finished building in ${(Date.now() - st) / 1000}s`);
|
|
} else {
|
|
console.log(`Worker done in ${(Date.now() - st) / 1000}s`);
|
|
}
|
|
}
|
|
|
|
build();
|